@alexkroman1/aai 1.2.3 → 1.3.1

package/host/providers/stt/assemblyai.ts

@@ -0,0 +1,154 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * AssemblyAI Universal-Streaming STT adapter.
+ *
+ * Wraps the `assemblyai` Node SDK's {@link StreamingTranscriber} and
+ * normalizes its event surface onto the {@link SttProvider} /
+ * {@link SttEvents} contract consumed by the pipeline orchestrator.
+ *
+ * Default model: `"u3pro-rt"` (Universal-3 Pro Real-Time). The adapter
+ * maps that to the SDK's `"u3-rt-pro"` `speechModel` value; any other
+ * string is forwarded verbatim.
+ */
+
+import { AssemblyAI, type StreamingTranscriber } from "assemblyai";
+import { createNanoEvents, type Emitter } from "nanoevents";
+import type {
+  SttError,
+  SttEvents,
+  SttOpenOptions,
+  SttProvider,
+  SttSession,
+} from "../../../sdk/providers.ts";
+
+export interface AssemblyAIOptions {
+  /**
+   * Streaming speech model. Defaults to `"u3pro-rt"` (Universal-3 Pro
+   * Real-Time). Arbitrary strings are forwarded to the SDK unchanged.
+   */
+  model?: "u3pro-rt" | string;
+  /**
+   * AssemblyAI API key. Falls back to `SttOpenOptions.apiKey`, then
+   * `process.env.ASSEMBLYAI_API_KEY`.
+   */
+  apiKey?: string;
+}
+
+/** Internal: SttSession with a test-only handle to the raw SDK transcriber. */
+export interface AssemblyAISession extends SttSession {
+  /** @internal Test-only: exposes the underlying SDK transcriber for fixture replay. */
+  readonly _transcriber: StreamingTranscriber;
+}
+
+/** Translate the adapter's model alias to the SDK's `speechModel` value. */
+function resolveSpeechModel(model: string): string {
+  // Plan's public name is "u3pro-rt"; the SDK's enum uses "u3-rt-pro".
+  if (model === "u3pro-rt") return "u3-rt-pro";
+  return model;
+}
+
+function makeError(message: string): SttError {
+  const err = new Error(message) as SttError & { code: SttError["code"] };
+  (err as { code: SttError["code"] }).code = "stt_stream_error";
+  return err;
+}
+
+export function assemblyAI(opts: AssemblyAIOptions = {}): SttProvider {
+  return {
+    name: "assemblyai",
+    async open(openOpts: SttOpenOptions): Promise<SttSession> {
+      const apiKey = opts.apiKey ?? openOpts.apiKey ?? process.env.ASSEMBLYAI_API_KEY;
+      if (!apiKey) {
+        const err = new Error(
+          "AssemblyAI STT adapter: missing API key. Provide via the factory option, SttOpenOptions, or the ASSEMBLYAI_API_KEY environment variable.",
+        ) as SttError & { code: SttError["code"] };
+        (err as { code: SttError["code"] }).code = "stt_auth_failed";
+        throw err;
+      }
+
+      const client = new AssemblyAI({ apiKey });
+      const speechModel = resolveSpeechModel(opts.model ?? "u3pro-rt");
+      const transcriber = client.streaming.transcriber({
+        sampleRate: openOpts.sampleRate,
+        // The SDK types `speechModel` as a string-literal union; the adapter
+        // accepts `string` as an escape hatch, so cast at the boundary.
+        speechModel: speechModel as never,
+        ...(openOpts.sttPrompt ? { prompt: openOpts.sttPrompt } : {}),
+      });
+
+      const emitter: Emitter<SttEvents> = createNanoEvents<SttEvents>();
+      let closed = false;
+
+      transcriber.on("turn", (event) => {
+        if (closed) return;
+        const text = event.transcript ?? "";
+        if (event.end_of_turn) {
+          if (text.length > 0) emitter.emit("final", text);
+        } else if (text.length > 0) {
+          emitter.emit("partial", text);
+        }
+      });
+
+      transcriber.on("error", (err) => {
+        if (closed) return;
+        emitter.emit("error", makeError(err?.message ?? String(err)));
+      });
+
+      transcriber.on("close", (code) => {
+        if (closed) return;
+        // 1000 = normal closure.
+        if (code !== 1000) {
+          emitter.emit("error", makeError(`socket closed ${code}`));
+        }
+      });
+
+      try {
+        await transcriber.connect();
+      } catch (cause) {
+        const err = new Error(
+          `AssemblyAI STT: connect failed: ${cause instanceof Error ? cause.message : String(cause)}`,
+        ) as SttError & { code: SttError["code"] };
+        (err as { code: SttError["code"] }).code = "stt_connect_failed";
+        throw err;
+      }
+
+      const close = async (): Promise<void> => {
+        if (closed) return;
+        closed = true;
+        try {
+          await transcriber.close();
+        } catch {
+          // Swallow: the caller has already decided to tear down.
+        }
+      };
+
+      // Wire session-level abort to close the SDK socket.
+      if (openOpts.signal.aborted) {
+        void close();
+      } else {
+        openOpts.signal.addEventListener("abort", () => void close(), {
+          once: true,
+        });
+      }
+
+      const session: AssemblyAISession = {
+        sendAudio(pcm: Int16Array) {
+          if (closed) return;
+          // The SDK's sendAudio accepts ArrayBufferLike. Forward a detached
+          // copy of the PCM view's window so the consumer sees only this
+          // chunk's bytes.
+          const copy = new Uint8Array(pcm.byteLength);
+          copy.set(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength));
+          transcriber.sendAudio(copy.buffer);
+        },
+        on(event, fn) {
+          return emitter.on(event, fn);
+        },
+        close,
+        _transcriber: transcriber,
+      };
+
+      return session;
+    },
+  };
+}

package/host/providers/stt/fixtures/assemblyai/basic-turn.json

@@ -0,0 +1,30 @@
+[
+  { "type": "Begin", "id": "sess-x", "expires_at": 0 },
+  {
+    "type": "Turn",
+    "turn_order": 1,
+    "turn_is_formatted": false,
+    "end_of_turn": false,
+    "transcript": "what",
+    "end_of_turn_confidence": 0.1,
+    "words": []
+  },
+  {
+    "type": "Turn",
+    "turn_order": 1,
+    "turn_is_formatted": false,
+    "end_of_turn": false,
+    "transcript": "what's the",
+    "end_of_turn_confidence": 0.3,
+    "words": []
+  },
+  {
+    "type": "Turn",
+    "turn_order": 1,
+    "turn_is_formatted": true,
+    "end_of_turn": true,
+    "transcript": "what's the weather?",
+    "end_of_turn_confidence": 0.95,
+    "words": []
+  }
+]

package/host/providers/stt-barrel.ts

@@ -0,0 +1,13 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * `@alexkroman1/aai/stt` subpath barrel. Re-exports the STT provider
+ * contract types (via `stt.ts` → `sdk/providers.ts`) alongside the
+ * concrete AssemblyAI adapter factory. Task 9 owns wiring this file
+ * into `package.json` exports.
+ */
+
+// biome-ignore lint/performance/noReExportAll: subpath barrel
+export * from "./stt/assemblyai.ts";
+// Type-only re-export — no biome suppression needed; `export type *` is
+// excluded from the `noReExportAll` rule.
+export type * from "./stt.ts";

package/host/providers/stt.ts

@@ -0,0 +1,3 @@
+// Copyright 2025 the AAI authors. MIT license.
+/** STT provider interface — re-exported from sdk/ for host-side consumption. */
+export type * from "../../sdk/providers.ts";

package/host/providers/tts/cartesia.test.ts

@@ -0,0 +1,210 @@
+// Copyright 2025 the AAI authors. MIT license.
+/** Unit test for the Cartesia TTS adapter. Mocks `@cartesia/cartesia-js`. */
+
+import { beforeEach, describe, expect, test, vi } from "vitest";
+import { flush } from "../../_test-utils.ts";
+import { type CartesiaSession, cartesia } from "./cartesia.ts";
+
+// Recorded interactions on the fake `TTSWSContext` — one entry per method call.
+interface RecordedSend {
+  kind: "send" | "cancel";
+  contextId: string;
+  transcript?: string | undefined;
+  continue?: boolean | undefined;
+  language?: string | undefined;
+  model_id?: string | undefined;
+}
+
+const sends: RecordedSend[] = [];
+
+/** Minimal shape of the request the adapter sends to Cartesia. */
+interface FakeGenerationRequest {
+  transcript: string;
+  continue: boolean;
+  language?: string;
+  model_id?: string;
+}
+
+/**
+ * Fake `TTSWSContext`. Mirrors the fields the adapter touches:
+ * `contextId`, `send`, `cancel`.
+ */
+interface FakeContext {
+  contextId: string;
+  send(req: FakeGenerationRequest): Promise<void>;
+  cancel(): Promise<void>;
+}
+
+/** Fake `TTSWS`. EventEmitter-ish with a `_fire` test hook. */
+interface FakeTTSWS {
+  contexts: FakeContext[];
+  context(opts: { contextId: string }): FakeContext;
+  on(event: string, fn: (...args: unknown[]) => void): FakeTTSWS;
+  close(props?: { code: number; reason: string }): void;
+  _fire(event: string, payload: unknown): void;
+}
+
+vi.mock("@cartesia/cartesia-js", () => {
+  const makeWs = (): FakeTTSWS => {
+    const listeners = new Map<string, Array<(...args: unknown[]) => void>>();
+    const ws: FakeTTSWS = {
+      contexts: [],
+      context(opts) {
+        const ctx: FakeContext = {
+          contextId: opts.contextId,
+          async send(req) {
+            sends.push({
+              kind: "send",
+              contextId: ctx.contextId,
+              transcript: req.transcript,
+              continue: req.continue,
+              language: req.language,
+              model_id: req.model_id,
+            });
+          },
+          async cancel() {
+            sends.push({ kind: "cancel", contextId: ctx.contextId });
+          },
+        };
+        ws.contexts.push(ctx);
+        return ctx;
+      },
+      on(event, fn) {
+        const arr = listeners.get(event) ?? [];
+        arr.push(fn);
+        listeners.set(event, arr);
+        return ws;
+      },
+      close(_props) {
+        /* no-op */
+      },
+      _fire(event, payload) {
+        for (const fn of listeners.get(event) ?? []) fn(payload);
+      },
+    };
+    return ws;
+  };
+  return {
+    Cartesia: class {
+      tts = {
+        websocket: async () => makeWs(),
+      };
+    },
+  };
+});
+
+beforeEach(() => {
+  sends.length = 0;
+});
+
+async function openSession(): Promise<{
+  session: CartesiaSession;
+  controller: AbortController;
+}> {
+  const provider = cartesia({ voice: "voice-id", apiKey: "k" });
+  const controller = new AbortController();
+  const session = (await provider.open({
+    sampleRate: 16_000,
+    apiKey: "k",
+    signal: controller.signal,
+  })) as CartesiaSession;
+  return { session, controller };
+}
+
+describe("cartesia TTS adapter", () => {
+  test("sendText deltas share one contextId; flush ends the turn; next turn uses a fresh contextId", async () => {
+    const { session, controller } = await openSession();
+    const turn1 = session._currentContextId();
+
+    session.sendText("hello");
+    session.sendText(" world");
+    session.flush();
+    await flush();
+
+    // All three sends for turn 1 carry the same contextId — two deltas with
+    // continue: true, then an empty-transcript send with continue: false.
+    const turn1Sends = sends.filter((s) => s.contextId === turn1);
+    expect(turn1Sends).toEqual([
+      {
+        kind: "send",
+        contextId: turn1,
+        transcript: "hello",
+        continue: true,
+        language: "en",
+        model_id: "sonic-2",
+      },
+      {
+        kind: "send",
+        contextId: turn1,
+        transcript: " world",
+        continue: true,
+        language: "en",
+        model_id: "sonic-2",
+      },
+      {
+        kind: "send",
+        contextId: turn1,
+        transcript: "",
+        continue: false,
+        language: "en",
+        model_id: "sonic-2",
+      },
+    ]);
+
+    // After flush(), the adapter has rotated to a new context.
+    const turn2 = session._currentContextId();
+    expect(turn2).not.toBe(turn1);
+
+    // Subsequent sendText targets the new context.
+    session.sendText("next");
+    await flush();
+    expect(sends.filter((s) => s.contextId === turn2)).toEqual([
+      {
+        kind: "send",
+        contextId: turn2,
+        transcript: "next",
+        continue: true,
+        language: "en",
+        model_id: "sonic-2",
+      },
+    ]);
+
+    controller.abort();
+    await session.close();
+  });
+
+  test("cancel() calls ws.cancelContext(contextId) and emits `done` synchronously", async () => {
+    const { session, controller } = await openSession();
+    const turn1 = session._currentContextId();
+
+    const doneEvents: number[] = [];
+    session.on("done", () => doneEvents.push(Date.now()));
+
+    session.sendText("hello");
+    // cancel() must emit `done` synchronously — the orchestrator advances
+    // state on `done`, and barge-in response cannot be microtask-deferred.
+    session.cancel();
+    expect(doneEvents.length).toBe(1);
+
+    await flush();
+
+    // We expect: send("hello", continue:true) on turn1, then cancel(turn1).
+    expect(sends).toEqual([
+      {
+        kind: "send",
+        contextId: turn1,
+        transcript: "hello",
+        continue: true,
+        language: "en",
+        model_id: "sonic-2",
+      },
+      { kind: "cancel", contextId: turn1 },
+    ]);
+
+    // Cancelling rotates the context so the next turn is unambiguous.
+    expect(session._currentContextId()).not.toBe(turn1);
+
+    controller.abort();
+    await session.close();
+  });
+});

package/host/providers/tts/cartesia.ts

@@ -0,0 +1,251 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * Cartesia TTS adapter — streaming WebSocket with per-turn `context_id`.
+ *
+ * Wraps `@cartesia/cartesia-js`'s `TTSWS` / `TTSWSContext` and normalizes it
+ * onto the {@link TtsProvider} / {@link TtsEvents} contract consumed by the
+ * pipeline orchestrator.
+ *
+ * **Per-turn context lifecycle.** Each `sendText(...)` within the same turn
+ * appends to the same Cartesia context. On `flush()` or `cancel()`, a new
+ * context is minted for the next turn — so concurrent `cancel({ contextId })`
+ * only targets the in-flight turn, never the one that follows.
+ *
+ * **Audio format.** The adapter requests `raw` / `pcm_s16le` at the
+ * negotiated `sampleRate` so it can forward chunks as `Int16Array` with no
+ * conversion.
+ */
+
+import { randomUUID } from "node:crypto";
+import { Cartesia } from "@cartesia/cartesia-js";
+import type { TTSWS, TTSWSContext } from "@cartesia/cartesia-js/resources/tts";
+import { createNanoEvents, type Emitter } from "nanoevents";
+import type {
+  TtsError,
+  TtsEvents,
+  TtsOpenOptions,
+  TtsProvider,
+  TtsSession,
+} from "../../../sdk/providers.ts";
+
+export interface CartesiaOptions {
+  /** Cartesia voice ID. Required. */
+  voice: string;
+  /** Model ID. Defaults to `"sonic-2"`. */
+  model?: string;
+  /**
+   * Cartesia API key. Falls back to `TtsOpenOptions.apiKey`, then
+   * `process.env.CARTESIA_API_KEY`.
+   */
+  apiKey?: string;
+  /** Spoken language hint. Defaults to `"en"`. */
+  language?: string;
+}
+
+/** Internal: TtsSession with a test-only handle to the raw SDK socket. */
+export interface CartesiaSession extends TtsSession {
+  /** @internal Test-only: exposes the underlying SDK WebSocket wrapper. */
+  readonly _ws: TTSWS;
+  /** @internal Test-only: id of the currently-active context. */
+  readonly _currentContextId: () => string;
+}
+
+function makeError(message: string): TtsError {
+  const err = new Error(message) as TtsError & { code: TtsError["code"] };
+  (err as { code: TtsError["code"] }).code = "tts_stream_error";
+  return err;
+}
+
+/** PCM16 sample rates supported by Cartesia's `raw` output format. */
+const CARTESIA_PCM16_RATES = [
+  8000, 16_000, 22_050, 24_000, 44_100, 48_000,
+] as const satisfies readonly number[];
+type CartesiaSampleRate = (typeof CARTESIA_PCM16_RATES)[number];
+
+function assertSupportedSampleRate(rate: number): CartesiaSampleRate {
+  if ((CARTESIA_PCM16_RATES as readonly number[]).includes(rate)) {
+    return rate as CartesiaSampleRate;
+  }
+  const err = new Error(
+    `Cartesia TTS adapter: unsupported sample rate ${rate}. Supported: ${CARTESIA_PCM16_RATES.join(", ")}.`,
+  ) as TtsError & { code: TtsError["code"] };
+  (err as { code: TtsError["code"] }).code = "tts_connect_failed";
+  throw err;
+}
+
+export function cartesia(opts: CartesiaOptions): TtsProvider {
+  return {
+    name: "cartesia",
+    async open(openOpts: TtsOpenOptions): Promise<TtsSession> {
+      const apiKey = opts.apiKey ?? openOpts.apiKey ?? process.env.CARTESIA_API_KEY;
+      if (!apiKey) {
+        const err = new Error(
+          "Cartesia TTS adapter: missing API key. Provide via the factory option, TtsOpenOptions, or the CARTESIA_API_KEY environment variable.",
+        ) as TtsError & { code: TtsError["code"] };
+        (err as { code: TtsError["code"] }).code = "tts_auth_failed";
+        throw err;
+      }
+
+      const sampleRate = assertSupportedSampleRate(openOpts.sampleRate);
+      const model = opts.model ?? "sonic-2";
+      const language = opts.language ?? "en";
+
+      const client = new Cartesia({ apiKey });
+      let ws: TTSWS;
+      try {
+        ws = await client.tts.websocket();
+      } catch (cause) {
+        const err = new Error(
+          `Cartesia TTS: connect failed: ${cause instanceof Error ? cause.message : String(cause)}`,
+        ) as TtsError & { code: TtsError["code"] };
+        (err as { code: TtsError["code"] }).code = "tts_connect_failed";
+        throw err;
+      }
+
+      const emitter: Emitter<TtsEvents> = createNanoEvents<TtsEvents>();
+      let closed = false;
+
+      /** Mint a fresh context bound to the shared TTSWS connection. */
+      const mintContext = (): TTSWSContext =>
+        ws.context({
+          model_id: model,
+          voice: { mode: "id", id: opts.voice },
+          output_format: {
+            container: "raw",
+            encoding: "pcm_s16le",
+            sample_rate: sampleRate,
+          },
+          contextId: randomUUID(),
+        });
+
+      let context = mintContext();
+      /**
+       * `doneEmitted` guards against emitting `done` more than once per turn.
+       * Reset whenever a fresh context is minted (i.e. at turn boundaries).
+       */
+      let doneEmitted = false;
+      const rotateContext = () => {
+        context = mintContext();
+        doneEmitted = false;
+      };
+      const emitDoneOnce = () => {
+        if (doneEmitted || closed) return;
+        doneEmitted = true;
+        emitter.emit("done");
+      };
+
+      // Route SDK events onto the adapter's event surface, filtering by the
+      // currently-active `context_id`. The TTSWS EventEmitter fires globally
+      // across all contexts on the socket; we only care about the active one.
+      ws.on("chunk", (event) => {
+        if (closed) return;
+        if (event.context_id !== context.contextId) return;
+        // SDK decodes base64 → Buffer on receipt (`event.audio`). Forward as
+        // Int16Array over the same byte window.
+        const buf = event.audio;
+        if (!buf || buf.byteLength === 0) return;
+        // Cartesia sends PCM16 little-endian with even byte counts. Be defensive.
+        const evenBytes = buf.byteLength - (buf.byteLength % 2);
+        if (evenBytes === 0) return;
+        const pcm = new Int16Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + evenBytes));
+        emitter.emit("audio", pcm);
+      });
+
+      ws.on("done", (event) => {
+        if (closed) return;
+        if (event.context_id !== context.contextId) return;
+        emitDoneOnce();
+      });
+
+      ws.on("error", (err) => {
+        if (closed) return;
+        emitter.emit("error", makeError(err?.message ?? String(err)));
+      });
+
+      const close = async (): Promise<void> => {
+        if (closed) return;
+        closed = true;
+        try {
+          ws.close({ code: 1000, reason: "client close" });
+        } catch {
+          // Swallow: caller has already decided to tear down.
+        }
+      };
+
+      // Session-level abort → close the SDK socket.
+      if (openOpts.signal.aborted) {
+        void close();
+      } else {
+        openOpts.signal.addEventListener("abort", () => void close(), {
+          once: true,
+        });
+      }
+
+      /** Static part of each generation request; only `transcript` and
+       * `continue` vary per send. Pinned here so `language` threads through. */
+      const baseRequest = {
+        model_id: model,
+        voice: { mode: "id" as const, id: opts.voice },
+        output_format: {
+          container: "raw" as const,
+          encoding: "pcm_s16le" as const,
+          sample_rate: sampleRate,
+        },
+        language,
+      };
+
+      /**
+       * Swallow rejections from async SDK calls — the global `error`
+       * listener on `ws` emits a normalized {@link TtsError}, so there's
+       * nothing useful for the caller to do with per-send failures.
+       */
+      const ignoreRejection = (_err: unknown): void => {
+        // intentionally empty
+      };
+
+      const session: CartesiaSession = {
+        sendText(text: string) {
+          if (closed || text.length === 0) return;
+          // Send a delta with `continue: true`, sharing the same
+          // context_id across all deltas of this turn.
+          void context
+            .send({ ...baseRequest, transcript: text, continue: true })
+            .catch(ignoreRejection);
+        },
+        flush() {
+          if (closed) return;
+          // Send an empty transcript with `continue: false` — the canonical
+          // end-of-turn signal. The server replies with a `done` event
+          // tagged with this context's id, which drives `emitDoneOnce`. We
+          // also microtask-emit `done` as a fallback so the orchestrator's
+          // state machine can't wedge if the server event is dropped.
+          // TODO: drop the microtask fallback once we've verified Cartesia
+          // always emits a `done` for cleanly-flushed contexts. See
+          // 2026-04-22-pluggable-providers-design.md → "Note on flush() timing".
+          void context
+            .send({ ...baseRequest, transcript: "", continue: false })
+            .catch(ignoreRejection);
+          queueMicrotask(emitDoneOnce);
+          rotateContext();
+        },
+        cancel() {
+          if (closed) return;
+          // `cancel()` calls ws.cancelContext(contextId) under the hood.
+          void context.cancel().catch(ignoreRejection);
+          // Emit `done` synchronously — the orchestrator's state machine
+          // advances on `done`, and barge-in must not be delayed.
+          emitDoneOnce();
+          rotateContext();
+        },
+        on(event, fn) {
+          return emitter.on(event, fn);
+        },
+        close,
+        _ws: ws,
+        _currentContextId: () => context.contextId,
+      };
+
+      return session;
+    },
+  };
+}

package/host/providers/tts-barrel.ts

@@ -0,0 +1,13 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * `@alexkroman1/aai/tts` subpath barrel. Re-exports the TTS provider
+ * contract types (via `tts.ts` → `sdk/providers.ts`) alongside the
+ * concrete Cartesia adapter factory. Task 9 owns wiring this file
+ * into `package.json` exports.
+ */
+
+// biome-ignore lint/performance/noReExportAll: subpath barrel
+export * from "./tts/cartesia.ts";
+// Type-only re-export — no biome suppression needed; `export type *` is
+// excluded from the `noReExportAll` rule.
+export type * from "./tts.ts";

package/host/providers/tts.ts

@@ -0,0 +1,3 @@
+// Copyright 2025 the AAI authors. MIT license.
+/** TTS provider interface — re-exported from sdk/ for host-side consumption. */
+export type * from "../../sdk/providers.ts";