@alexkroman1/aai 1.2.3 → 1.3.1

package/host/_pipeline-test-fakes.ts

@@ -0,0 +1,357 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * In-memory fake providers + fake `LanguageModel` for pipeline-session tests.
+ *
+ * These fakes do not touch the network. Each `createFake*Provider()` returns a
+ * provider whose `open()` records the most recently opened session so tests
+ * can reach into it via `.last()` and drive events (partial/final transcripts,
+ * TTS chunks) or observe calls (`sendText`, `flush`, `cancel`).
+ *
+ * The fake `LanguageModel` implements the minimum of {@link LanguageModelV3}
+ * required by `streamText` — `doStream()` returns a `ReadableStream` of
+ * {@link LanguageModelV3StreamPart}s produced from a scripted sequence.
+ *
+ * @internal Not part of the public API.
+ */
+
+import type { LanguageModel } from "ai";
+import { createNanoEvents, type Emitter } from "nanoevents";
+import { vi } from "vitest";
+import type {
+  SttEvents,
+  SttOpenOptions,
+  SttProvider,
+  SttSession,
+  TtsEvents,
+  TtsOpenOptions,
+  TtsProvider,
+  TtsSession,
+} from "../sdk/providers.ts";
+
+// ─── Fake STT ───────────────────────────────────────────────────────────────
+
+export type FakeSttSession = SttSession & {
+  readonly emitter: Emitter<SttEvents>;
+  readonly opts: SttOpenOptions;
+  readonly audioFrames: Int16Array[];
+  readonly closed: { value: boolean };
+  firePartial(text: string): void;
+  fireFinal(text: string): void;
+  fireError(
+    code: "stt_stream_error" | "stt_connect_failed" | "stt_auth_failed",
+    message: string,
+  ): void;
+};
+
+export type FakeSttProvider = SttProvider & {
+  /** The most recently opened session, or undefined if `open()` hasn't been called. */
+  last(): FakeSttSession | undefined;
+  readonly sessions: FakeSttSession[];
+};
+
+export function createFakeSttProvider(): FakeSttProvider {
+  const sessions: FakeSttSession[] = [];
+  return {
+    name: "fake-stt",
+    sessions,
+    last: () => sessions.at(-1),
+    async open(opts: SttOpenOptions): Promise<SttSession> {
+      const emitter = createNanoEvents<SttEvents>();
+      const audioFrames: Int16Array[] = [];
+      const closed = { value: false };
+      const session: FakeSttSession = {
+        emitter,
+        opts,
+        audioFrames,
+        closed,
+        sendAudio: vi.fn((pcm: Int16Array) => {
+          audioFrames.push(pcm);
+        }),
+        on: emitter.on.bind(emitter) as SttSession["on"],
+        close: vi.fn(async () => {
+          closed.value = true;
+        }),
+        firePartial(text: string) {
+          emitter.emit("partial", text);
+        },
+        fireFinal(text: string) {
+          emitter.emit("final", text);
+        },
+        fireError(code, message) {
+          const err = Object.assign(new Error(message), { code }) as Parameters<
+            SttEvents["error"]
+          >[0];
+          emitter.emit("error", err);
+        },
+      };
+      sessions.push(session);
+      return session;
+    },
+  };
+}
+
+// ─── Fake TTS ───────────────────────────────────────────────────────────────
+
+export type FakeTtsSession = TtsSession & {
+  readonly emitter: Emitter<TtsEvents>;
+  readonly opts: TtsOpenOptions;
+  readonly textChunks: string[];
+  readonly closed: { value: boolean };
+  readonly sendText: ReturnType<typeof vi.fn<(text: string) => void>>;
+  readonly flush: ReturnType<typeof vi.fn<() => void>>;
+  readonly cancel: ReturnType<typeof vi.fn<() => void>>;
+  fireAudio(pcm: Int16Array): void;
+  fireError(
+    code: "tts_stream_error" | "tts_connect_failed" | "tts_auth_failed",
+    message: string,
+  ): void;
+};
+
+export type FakeTtsProvider = TtsProvider & {
+  /** The most recently opened session, or undefined if `open()` hasn't been called. */
+  last(): FakeTtsSession | undefined;
+  readonly sessions: FakeTtsSession[];
+};
+
+/**
+ * Fake TTS provider. By default, `flush()` synchronously emits a single `done`
+ * event so tests don't have to script the drain separately. Pass
+ * `{ autoDoneOnFlush: false }` to drive `done` manually.
+ */
+export function createFakeTtsProvider(
+  options: { autoDoneOnFlush?: boolean } = {},
+): FakeTtsProvider {
+  const autoDoneOnFlush = options.autoDoneOnFlush ?? true;
+  const sessions: FakeTtsSession[] = [];
+  return {
+    name: "fake-tts",
+    sessions,
+    last: () => sessions.at(-1),
+    async open(opts: TtsOpenOptions): Promise<TtsSession> {
+      const emitter = createNanoEvents<TtsEvents>();
+      const textChunks: string[] = [];
+      const closed = { value: false };
+      const sendText = vi.fn((text: string) => {
+        textChunks.push(text);
+      });
+      const flush = vi.fn(() => {
+        if (autoDoneOnFlush) emitter.emit("done");
+      });
+      const cancel = vi.fn(() => {
+        emitter.emit("done");
+      });
+      const session: FakeTtsSession = {
+        emitter,
+        opts,
+        textChunks,
+        closed,
+        sendText,
+        flush,
+        cancel,
+        on: emitter.on.bind(emitter) as TtsSession["on"],
+        close: vi.fn(async () => {
+          closed.value = true;
+        }),
+        fireAudio(pcm: Int16Array) {
+          emitter.emit("audio", pcm);
+        },
+        fireError(code, message) {
+          const err = Object.assign(new Error(message), { code }) as Parameters<
+            TtsEvents["error"]
+          >[0];
+          emitter.emit("error", err);
+        },
+      };
+      sessions.push(session);
+      return session;
+    },
+  };
+}
+
+/**
+ * Fake STT provider that throws on `open()` with a given error code. Used to
+ * test atomic provider open — TTS should not be opened at all when STT fails.
+ */
+export function createFailingSttProvider(
+  code: "stt_connect_failed" | "stt_auth_failed" | "stt_stream_error",
+  message: string,
+): SttProvider {
+  return {
+    name: "failing-stt",
+    async open(): Promise<SttSession> {
+      const err = Object.assign(new Error(message), { code }) as Error & { code: typeof code };
+      throw err;
+    },
+  };
+}
+
+/**
+ * Fake TTS provider that throws on `open()` with a given error code. Used to
+ * test atomic provider open — STT should be closed when TTS fails.
+ */
+export function createFailingTtsProvider(
+  code: "tts_connect_failed" | "tts_auth_failed" | "tts_stream_error",
+  message: string,
+): TtsProvider {
+  return {
+    name: "failing-tts",
+    async open(): Promise<TtsSession> {
+      const err = Object.assign(new Error(message), { code }) as Error & { code: typeof code };
+      throw err;
+    },
+  };
+}
+
+// ─── Fake LLM ───────────────────────────────────────────────────────────────
+
+/**
+ * A scripted stream part. `text` yields a `text-delta` in the LLM provider's
+ * raw wire format; `tool-call` / `tool-result` emit the corresponding parts
+ * (v3 provider spec: `toolCallId`, `toolName`, `input` as JSON string for
+ * calls, `result` as JSON-serialisable value for results).
+ */
+export type ScriptedPart =
+  | { type: "text"; text: string }
+  | { type: "tool-call"; toolCallId: string; toolName: string; input: string }
+  | { type: "tool-result"; toolCallId: string; toolName: string; result: unknown }
+  | { type: "error"; error: unknown };
+
+/**
+ * Shape of the single stream part yielded by an LLM provider's `doStream()`.
+ * This is a loose local definition — the real type lives in `@ai-sdk/provider`
+ * as `LanguageModelV3StreamPart`, but we don't want a direct dependency on
+ * that package. The test fakes only need enough of the shape that the
+ * `ai` package's `streamText` will forward through to consumers.
+ */
+type StreamPart =
+  | { type: "stream-start"; warnings: never[] }
+  | { type: "text-start"; id: string }
+  | { type: "text-delta"; id: string; delta: string }
+  | { type: "text-end"; id: string }
+  | { type: "tool-call"; toolCallId: string; toolName: string; input: string }
+  | { type: "tool-result"; toolCallId: string; toolName: string; result: NonNullable<unknown> }
+  | { type: "error"; error: unknown }
+  | {
+      type: "finish";
+      usage: { inputTokens: number; outputTokens: number; totalTokens: number };
+      finishReason: string;
+    };
+
+function scriptedPartToStreamPart(part: ScriptedPart, textId: string): StreamPart {
+  switch (part.type) {
+    case "text":
+      return { type: "text-delta", id: textId, delta: part.text };
+    case "tool-call":
+      return {
+        type: "tool-call",
+        toolCallId: part.toolCallId,
+        toolName: part.toolName,
+        input: part.input,
+      };
+    case "tool-result":
+      return {
+        type: "tool-result",
+        toolCallId: part.toolCallId,
+        toolName: part.toolName,
+        result: part.result as NonNullable<unknown>,
+      };
+    case "error":
+      return { type: "error", error: part.error };
+    default: {
+      const never: never = part;
+      return never;
+    }
+  }
+}
+
+/** Wait `ms` or resolve immediately when `signal` aborts. */
+function delayOrAbort(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise<void>((resolve) => {
+    const t = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(t);
+        resolve();
+      },
+      { once: true },
+    );
+  });
+}
+
+async function streamScript(
+  controller: ReadableStreamDefaultController<StreamPart>,
+  script: ScriptedPart[],
+  delayMs: number | undefined,
+  signal: AbortSignal | undefined,
+): Promise<void> {
+  const textId = "text-1";
+  controller.enqueue({ type: "stream-start", warnings: [] });
+  controller.enqueue({ type: "text-start", id: textId });
+  try {
+    for (const part of script) {
+      if (signal?.aborted) break;
+      if (delayMs !== undefined && delayMs > 0) await delayOrAbort(delayMs, signal);
+      if (signal?.aborted) break;
+      controller.enqueue(scriptedPartToStreamPart(part, textId));
+    }
+  } finally {
+    controller.enqueue({ type: "text-end", id: textId });
+    controller.enqueue({
+      type: "finish",
+      usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+      finishReason: signal?.aborted ? "other" : "stop",
+    });
+    controller.close();
+  }
+}
+
+/**
+ * Create a fake {@link LanguageModel} that yields a scripted sequence of
+ * parts when `streamText` drives `doStream()`. The fake ignores the prompt
+ * and tools — it simply replays the script.
+ *
+ * Pass `{ delayMs: N }` to space out parts with `setTimeout(N)` so that
+ * barge-in tests can abort mid-stream deterministically.
+ *
+ * Pass `{ steps: ScriptedPart[][] }` (instead of `script`) for multi-step
+ * scenarios: each call to `doStream()` consumes the next step's parts.
+ * This is how `streamText` drives multi-turn tool loops under `stopWhen`.
+ *
+ * The returned value is cast to the `LanguageModel` union because we
+ * implement the provider shape structurally rather than importing the
+ * full `@ai-sdk/provider` types into the aai package.
+ */
+export function createFakeLanguageModel(
+  options:
+    | { script: ScriptedPart[]; delayMs?: number }
+    | { steps: ScriptedPart[][]; delayMs?: number },
+): LanguageModel {
+  const delayMs = options.delayMs;
+  const steps: ScriptedPart[][] = "steps" in options ? options.steps : [options.script];
+  let stepIndex = 0;
+  const model = {
+    specificationVersion: "v3" as const,
+    provider: "fake-llm",
+    modelId: "fake-llm-1",
+    supportedUrls: {} as Record<string, RegExp[]>,
+    async doGenerate(): Promise<never> {
+      throw new Error("fake LLM: doGenerate not implemented");
+    },
+    async doStream(opts: { abortSignal?: AbortSignal }): Promise<{
+      stream: ReadableStream<StreamPart>;
+    }> {
+      // Advance one step per call; after the last scripted step, keep
+      // yielding an empty step so an unexpected extra call completes cleanly.
+      const current = steps[stepIndex] ?? [];
+      stepIndex++;
+      const stream = new ReadableStream<StreamPart>({
+        start(controller) {
+          void streamScript(controller, current, delayMs, opts.abortSignal);
+        },
+      });
+      return { stream };
+    },
+  };
+  return model as unknown as LanguageModel;
+}

package/host/_test-utils.ts

@@ -81,6 +81,7 @@ export function makeMockHandle(): MockS2sHandle {
   return {
     on: emitter.on.bind(emitter),
     sendAudio: vi.fn(),
+    sendAudioRaw: vi.fn(),
     sendToolResult: vi.fn(),
     updateSession: vi.fn(),
     resumeSession: vi.fn(),

package/host/integration/fixtures/README.md

@@ -0,0 +1,49 @@
+# Integration Test Fixtures
+
+## hello-how-are-you.pcm16
+
+Required by `pipeline-reference.integration.test.ts`. A ~2-second monoaural
+16 kHz signed 16-bit little-endian PCM file of a voice saying "hello, how
+are you?" (or any similar short question the LLM can respond to).
+
+### Generating it
+
+Easiest: record yourself via the macOS Terminal using `sox`:
+
+```sh
+sox -d -r 16000 -c 1 -b 16 -e signed-integer hello-how-are-you.pcm16 trim 0 2
+```
+
+Or export from a DAW as raw PCM16 mono @ 16 kHz.
+
+Or use a CLI TTS to synthesize it (e.g. the same Cartesia voice we use at
+runtime, piped through `ffmpeg -ar 16000 -ac 1 -f s16le`):
+
+```sh
+# sketch — adjust to your preferred TTS
+curl -X POST https://api.cartesia.ai/tts/bytes ... \
+  | ffmpeg -i pipe:0 -ar 16000 -ac 1 -f s16le hello-how-are-you.pcm16
+```
+
+### Why it's not checked in
+
+Audio fixtures are binary blobs — we keep them out of the repo. Contributors
+with API keys + local fixtures can run the integration test; CI should
+provide both as secrets and generate/restore the fixture from a secure
+artifact store before the integration job runs.
+
+### Running the test
+
+```sh
+export ASSEMBLYAI_API_KEY=...
+export OPENAI_API_KEY=...
+export CARTESIA_API_KEY=...
+VITEST_PROFILE=integration \
+  VITEST_INCLUDE='host/integration/**/*.integration.test.ts' \
+  pnpm --filter @alexkroman1/aai exec vitest run \
+    -c ../../vitest.slow.config.ts
+```
+
+If the fixture is missing, the test throws a clear error pointing here. If
+any of the env vars are missing, the whole suite is skipped via
+`describe.skipIf`.

package/host/integration/pipeline-reference.integration.test.ts

@@ -0,0 +1,124 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * Integration test for the pluggable-providers pipeline reference stack.
+ *
+ * Runs only when VITEST_PROFILE=integration is set AND all three API keys
+ * plus the input audio fixture are available. Exercises the full STT → LLM → TTS
+ * path with real providers (AssemblyAI u3pro-rt + OpenAI gpt-4o-mini +
+ * Cartesia) so latency and wire-format issues are caught before release.
+ *
+ * To run locally:
+ *
+ *   export ASSEMBLYAI_API_KEY=...
+ *   export OPENAI_API_KEY=...
+ *   export CARTESIA_API_KEY=...
+ *   VITEST_PROFILE=integration \
+ *     VITEST_INCLUDE=host/integration/**\/*.integration.test.ts \
+ *     pnpm --filter @alexkroman1/aai exec vitest run -c ../../vitest.slow.config.ts
+ *
+ * See fixtures/README.md for how to generate the required audio input.
+ */
+
+import { readFile, stat } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { openai } from "@ai-sdk/openai";
+import { describe, expect, test } from "vitest";
+import type { AgentConfig, ExecuteTool } from "../../sdk/_internal-types.ts";
+import type { ClientEvent, ClientSink } from "../../sdk/protocol.ts";
+import { createPipelineSession } from "../pipeline-session.ts";
+import { assemblyAI } from "../providers/stt/assemblyai.ts";
+import { cartesia } from "../providers/tts/cartesia.ts";
+import { consoleLogger } from "../runtime-config.ts";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const fixturePath = join(here, "fixtures/hello-how-are-you.pcm16");
+
+async function fixtureExists(): Promise<boolean> {
+  try {
+    const s = await stat(fixturePath);
+    return s.isFile() && s.size > 0;
+  } catch {
+    return false;
+  }
+}
+
+const envReady = Boolean(
+  process.env.VITEST_PROFILE === "integration" &&
+    process.env.ASSEMBLYAI_API_KEY &&
+    process.env.OPENAI_API_KEY &&
+    process.env.CARTESIA_API_KEY,
+);
+
+describe.skipIf(!envReady)("pipeline integration — reference stack", () => {
+  test("audio in → transcript, LLM reply, TTS audio out", async () => {
+    if (!(await fixtureExists())) {
+      throw new Error(
+        `Fixture not found at ${fixturePath}. ` +
+          "See fixtures/README.md for instructions on generating it.",
+      );
+    }
+    const pcm = await readFile(fixturePath);
+    const events: ClientEvent[] = [];
+    const audioOut: Uint8Array[] = [];
+    const client: ClientSink = {
+      open: true,
+      event: (e) => {
+        events.push(e);
+      },
+      playAudioChunk: (chunk) => {
+        audioOut.push(chunk);
+      },
+      playAudioDone: () => {
+        /* no-op: test asserts on audioOut chunks directly */
+      },
+    };
+
+    const agentConfig: AgentConfig = {
+      name: "int",
+      systemPrompt: "You reply in one short sentence.",
+      greeting: "",
+      maxSteps: 1,
+    };
+
+    const executeTool: ExecuteTool = async () => "";
+
+    const session = createPipelineSession({
+      id: "int-1",
+      agent: "pipeline-reference",
+      client,
+      agentConfig,
+      toolSchemas: [],
+      executeTool,
+      stt: assemblyAI({ model: "u3pro-rt" }),
+      llm: openai("gpt-4o-mini"),
+      tts: cartesia({ voice: "694f9389-aac1-45b6-b726-9d9369183238" }),
+      // biome-ignore lint/style/noNonNullAssertion: envReady guard ensures presence
+      sttApiKey: process.env.ASSEMBLYAI_API_KEY!,
+      // biome-ignore lint/style/noNonNullAssertion: envReady guard ensures presence
+      ttsApiKey: process.env.CARTESIA_API_KEY!,
+      logger: consoleLogger,
+    });
+
+    await session.start();
+    session.onAudioReady();
+
+    // Stream the PCM fixture in ~100ms chunks (16 kHz PCM16 → 3200 bytes/100ms).
+    const chunkBytes = 3200;
+    for (let i = 0; i < pcm.length; i += chunkBytes) {
+      const chunk = pcm.subarray(i, Math.min(i + chunkBytes, pcm.length));
+      session.onAudio(new Uint8Array(chunk));
+      await new Promise((r) => setTimeout(r, 100));
+    }
+    await session.waitForTurn();
+    await session.stop();
+
+    const userTranscript = events.find((e) => e.type === "user_transcript");
+    expect(userTranscript).toBeDefined();
+    expect(String((userTranscript as { text: string }).text).toLowerCase()).toContain(
+      "how are you",
+    );
+    expect(events.some((e) => e.type === "reply_done")).toBe(true);
+    expect(audioOut.length).toBeGreaterThan(0);
+  }, 60_000);
+});

package/host/pipeline-session-ctx.test.ts

@@ -0,0 +1,31 @@
+// Copyright 2025 the AAI authors. MIT license.
+import { describe, expect, test } from "vitest";
+import { buildPipelineCtx } from "./pipeline-session-ctx.ts";
+import { consoleLogger } from "./runtime-config.ts";
+
+const baseDeps = {
+  id: "sess-1",
+  agent: "demo",
+  client: { event: () => undefined, playAudioChunk: () => undefined } as never,
+  agentConfig: { name: "demo", systemPrompt: "", maxSteps: 3 } as never,
+  executeTool: (async () => "") as never,
+  log: consoleLogger,
+};
+
+describe("buildPipelineCtx", () => {
+  test("starts with null provider sessions", () => {
+    const ctx = buildPipelineCtx(baseDeps);
+    expect(ctx.stt).toBeNull();
+    expect(ctx.tts).toBeNull();
+  });
+
+  test("pushMessages + beginReply + cancelReply match S2S semantics", () => {
+    const ctx = buildPipelineCtx(baseDeps);
+    ctx.pushMessages({ role: "user", content: "hi" });
+    ctx.beginReply("r1");
+    expect(ctx.reply.currentReplyId).toBe("r1");
+    ctx.cancelReply();
+    expect(ctx.reply.currentReplyId).toBeNull();
+    expect(ctx.conversationMessages).toEqual([{ role: "user", content: "hi" }]);
+  });
+});

package/host/pipeline-session-ctx.ts

@@ -0,0 +1,36 @@
+// Copyright 2025 the AAI authors. MIT license.
+/** Pipeline session context — base ctx + STT/TTS session slots. */
+
+import type { AgentConfig, ExecuteTool } from "../sdk/_internal-types.ts";
+import type { ClientSink } from "../sdk/protocol.ts";
+import type { SttSession, TtsSession } from "../sdk/providers.ts";
+import type { Logger } from "./runtime-config.ts";
+import { _buildBaseCtx, type BaseSessionCtx } from "./session-ctx.ts";
+
+/**
+ * Pipeline session context — {@link BaseSessionCtx} plus STT/TTS provider
+ * session handles. Replaces the S2S `s2s` field with decoupled `stt` + `tts`
+ * slots so the pipeline orchestrator can drive independent providers.
+ */
+export type PipelineSessionCtx = BaseSessionCtx & {
+  stt: SttSession | null;
+  tts: TtsSession | null;
+};
+
+export function buildPipelineCtx(opts: {
+  id: string;
+  agent: string;
+  client: ClientSink;
+  agentConfig: AgentConfig;
+  executeTool: ExecuteTool;
+  log: Logger;
+  maxHistory?: number | undefined;
+}): PipelineSessionCtx {
+  // Mutate the base ctx in place rather than spreading into a new object —
+  // the helper methods close over the base ctx reference, so spreading would
+  // leave them writing to an orphan object.
+  const base = _buildBaseCtx(opts) as PipelineSessionCtx;
+  base.stt = null;
+  base.tts = null;
+  return base;
+}