@alexkroman1/aai 1.2.3 → 1.3.0

package/host/runtime.ts

@@ -14,9 +14,11 @@ import { DEFAULT_SHUTDOWN_TIMEOUT_MS } from "../sdk/constants.ts";
 import type { Kv } from "../sdk/kv.ts";
 import type { ClientSink } from "../sdk/protocol.ts";
 import { buildReadyConfig, type ReadyConfig } from "../sdk/protocol.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "../sdk/providers.ts";
 import type { AgentDef } from "../sdk/types.ts";
 import { toolError } from "../sdk/utils.ts";
 import { resolveAllBuiltins } from "./builtin-tools.ts";
+import { createPipelineSession } from "./pipeline-session.ts";
 import type { Logger, S2SConfig } from "./runtime-config.ts";
 import { consoleLogger, DEFAULT_S2S_CONFIG } from "./runtime-config.ts";
 import type { CreateS2sWebSocket } from "./s2s.ts";
@@ -58,6 +60,18 @@ function createLocalKv(): Kv {
   return createUnstorageKv({ storage: createStorage() });
 }
 
+/**
+ * Resolve an API key host-side for pipeline providers.
+ *
+ * Checks the agent's declared env first, then the host process env as a
+ * fallback. Returns `""` when absent — pipeline providers surface a clear
+ * `MissingCredentialsError` via their `open()` that the orchestrator
+ * converts to a `session.error` wire event.
+ */
+function resolveApiKey(envVar: string, env: Record<string, string>): string {
+  return env[envVar] ?? process.env[envVar] ?? "";
+}
+
 /**
  * Configuration for {@link createRuntime}.
  *
@@ -111,6 +125,22 @@ export type RuntimeOptions = {
    * their own fetch wrapper.
    */
   fetch?: typeof globalThis.fetch | undefined;
+  /**
+   * Pluggable STT provider. Must be set together with `llm` and `tts` to
+   * route sessions through the pipeline path; leave all three unset for
+   * the default AssemblyAI Streaming Speech-to-Speech (S2S) path.
+   */
+  stt?: SttProvider | undefined;
+  /**
+   * Pluggable LLM provider (Vercel AI SDK `LanguageModel`). Must be set
+   * together with `stt` and `tts` to route sessions through the pipeline path.
+   */
+  llm?: LlmProvider | undefined;
+  /**
+   * Pluggable TTS provider. Must be set together with `stt` and `llm` to
+   * route sessions through the pipeline path.
+   */
+  tts?: TtsProvider | undefined;
 };
 
 /**
@@ -160,6 +190,14 @@ export function createRuntime(opts: RuntimeOptions): Runtime {
     sessionStartTimeoutMs,
     shutdownTimeoutMs = DEFAULT_SHUTDOWN_TIMEOUT_MS,
   } = opts;
+  // Derive session mode from the provider triple: all three set ⇒ pipeline,
+  // none set ⇒ s2s. Anything in-between is a configuration error.
+  const providerCount =
+    (opts.stt != null ? 1 : 0) + (opts.llm != null ? 1 : 0) + (opts.tts != null ? 1 : 0);
+  if (providerCount !== 0 && providerCount !== 3) {
+    throw new Error("stt, llm, and tts must be set together");
+  }
+  const mode: "s2s" | "pipeline" = providerCount === 3 ? "pipeline" : "s2s";
   const agentConfig = toAgentConfig(agent);
   const sessions = new Map<string, Session>();
   const sinkMap = new Map<string, ClientSink>();
@@ -241,6 +279,29 @@ export function createRuntime(opts: RuntimeOptions): Runtime {
     resumeFrom?: string;
   }): Session {
     sinkMap.set(sessionOpts.id, sessionOpts.client);
+    if (mode === "pipeline") {
+      // biome-ignore lint/style/noNonNullAssertion: providerCount === 3 ⇒ all set
+      const stt = opts.stt!;
+      // biome-ignore lint/style/noNonNullAssertion: providerCount === 3 ⇒ all set
+      const llm = opts.llm!;
+      // biome-ignore lint/style/noNonNullAssertion: providerCount === 3 ⇒ all set
+      const tts = opts.tts!;
+      return createPipelineSession({
+        id: sessionOpts.id,
+        agent: sessionOpts.agent,
+        client: sessionOpts.client,
+        agentConfig,
+        toolSchemas,
+        toolGuidance,
+        executeTool,
+        stt,
+        llm,
+        tts,
+        sttApiKey: resolveApiKey("ASSEMBLYAI_API_KEY", env),
+        ttsApiKey: resolveApiKey("CARTESIA_API_KEY", env),
+        logger,
+      });
+    }
     const apiKey = env.ASSEMBLYAI_API_KEY ?? "";
     return createS2sSession({
       id: sessionOpts.id,

package/host/s2s.test.ts

@@ -66,6 +66,7 @@ describe("connectS2s", () => {
     expect(handle).toEqual(
       expect.objectContaining({
         sendAudio: expect.any(Function),
+        sendAudioRaw: expect.any(Function),
         sendToolResult: expect.any(Function),
         updateSession: expect.any(Function),
         resumeSession: expect.any(Function),
@@ -125,6 +126,24 @@ describe("connectS2s", () => {
     expect(raw.send).not.toHaveBeenCalled();
   });
 
+  test("sendAudioRaw forwards the exact string to the socket", async () => {
+    const { raw, handle } = await setupHandle();
+
+    const frame = '{"type":"input.audio","audio":"abc"}';
+    handle.sendAudioRaw(frame);
+
+    expect(raw.send).toHaveBeenCalledOnce();
+    expect(raw.send.mock.calls[0]?.[0]).toBe(frame);
+  });
+
+  test("sendAudioRaw is no-op when ws is not open", async () => {
+    const { raw, handle } = await setupHandle();
+    raw.readyState = 3; // CLOSED
+
+    handle.sendAudioRaw('{"type":"input.audio","audio":"abc"}');
+    expect(raw.send).not.toHaveBeenCalled();
+  });
+
   test("sendToolResult sends tool.result message", async () => {
     const { raw, handle } = await setupHandle();
 

package/host/s2s.ts

@@ -159,6 +159,11 @@ export type S2sEvents = {
 export type S2sHandle = {
   on<K extends keyof S2sEvents>(event: K, cb: S2sEvents[K]): Unsubscribe;
   sendAudio(audio: Uint8Array): void;
+  /**
+   * Send a pre-encoded audio wire frame. For perf-critical callers (load tests)
+   * that batch-encode up front. Skips logging; caller owns wire format.
+   */
+  sendAudioRaw(jsonFrame: string): void;
   sendToolResult(callId: string, result: string): void;
   updateSession(config: S2sSessionConfig): void;
   resumeSession(sessionId: string): void;
@@ -212,6 +217,11 @@ export function connectS2s(opts: ConnectS2sOptions): Promise<S2sHandle> {
         ws.send(`{"type":"input.audio","audio":"${uint8ToBase64(audio)}"}`);
       },
 
+      sendAudioRaw(jsonFrame: string): void {
+        if (ws.readyState !== WS_OPEN) return;
+        ws.send(jsonFrame);
+      },
+
       sendToolResult(callId: string, result: string): void {
         const msg = { type: "tool.result", call_id: callId, result };
         log.info("S2S >> tool.result", { call_id: callId, resultLength: result.length });

package/host/session-ctx.ts

@@ -30,15 +30,18 @@ export type SessionDeps = {
 };
 
 /**
- * Session context threaded through event handlers.
+ * Transport-agnostic session context shared by S2S and pipeline sessions.
+ *
+ * Owns reply lifecycle, conversation history (with sliding-window truncation),
+ * and per-turn tool-call step enforcement. Transport-specific fields (e.g.
+ * `s2s` for S2S, `stt`/`tts` for the pipeline) live on the extending types.
  *
  * Split into three layers:
  * - {@link SessionDeps} — immutable dependencies (set once)
  * - {@link ReplyState} via `reply` — per-reply mutable state (reset on beginReply/cancelReply)
- * - Remaining fields — connection, conversation, and lifecycle methods
+ * - Remaining fields — conversation and lifecycle methods
  */
-export type S2sSessionCtx = SessionDeps & {
-  s2s: S2sHandle | null;
+export type BaseSessionCtx = SessionDeps & {
   reply: ReplyState;
   turnPromise: Promise<void> | null;
   conversationMessages: Message[];
@@ -50,7 +53,14 @@ export type S2sSessionCtx = SessionDeps & {
   chainTurn(p: Promise<void>): void;
 };
 
-export function buildCtx(opts: {
+/**
+ * S2S session context — {@link BaseSessionCtx} plus the S2S WebSocket handle.
+ */
+export type S2sSessionCtx = BaseSessionCtx & {
+  s2s: S2sHandle | null;
+};
+
+export function _buildBaseCtx(opts: {
   id: string;
   agent: string;
   client: ClientSink;
@@ -58,12 +68,11 @@ export function buildCtx(opts: {
   executeTool: ExecuteTool;
   log: Logger;
   maxHistory?: number | undefined;
-}): S2sSessionCtx {
+}): BaseSessionCtx {
   const { agentConfig, log } = opts;
   const maxHistory = opts.maxHistory ?? DEFAULT_MAX_HISTORY;
-  const ctx: S2sSessionCtx = {
+  const ctx: BaseSessionCtx = {
     ...opts,
-    s2s: null,
     reply: { pendingTools: [], toolCallCount: 0, currentReplyId: null },
     turnPromise: null,
     conversationMessages: [],
@@ -105,3 +114,21 @@ export function buildCtx(opts: {
   };
   return ctx;
 }
+
+export function buildCtx(opts: {
+  id: string;
+  agent: string;
+  client: ClientSink;
+  agentConfig: AgentConfig;
+  executeTool: ExecuteTool;
+  log: Logger;
+  maxHistory?: number | undefined;
+}): S2sSessionCtx {
+  // Mutate the base ctx in place rather than spreading into a new object —
+  // the helper methods close over the base ctx reference, so spreading would
+  // leave them writing to an orphan object (e.g. `beginReply` would mutate
+  // the base `reply`, not the spread copy's `reply`).
+  const base = _buildBaseCtx(opts) as S2sSessionCtx;
+  base.s2s = null;
+  return base;
+}

package/host/to-vercel-tools.test.ts

@@ -0,0 +1,153 @@
+// Copyright 2025 the AAI authors. MIT license.
+import { describe, expect, test, vi } from "vitest";
+import type { ToolSchema } from "../sdk/_internal-types.ts";
+import { toVercelTools } from "./to-vercel-tools.ts";
+
+const schemas: ToolSchema[] = [
+  {
+    name: "get_weather",
+    description: "Look up the weather.",
+    parameters: {
+      type: "object",
+      properties: { city: { type: "string" } },
+      required: ["city"],
+    },
+  },
+];
+
+describe("toVercelTools", () => {
+  test("produces one Vercel AI SDK tool per schema, keyed by name", () => {
+    const executeTool = vi.fn(async () => "sunny");
+    const tools = toVercelTools(schemas, {
+      executeTool,
+      sessionId: "s1",
+      messages: () => [],
+    });
+    expect(Object.keys(tools)).toEqual(["get_weather"]);
+    expect(tools.get_weather).toMatchObject({
+      description: "Look up the weather.",
+    });
+  });
+
+  test("execute delegates to ctx.executeTool with (name, args, sessionId, messages)", async () => {
+    const executeTool = vi.fn(async () => "rainy");
+    const tools = toVercelTools(schemas, {
+      executeTool,
+      sessionId: "sess-42",
+      messages: () => [{ role: "user", content: "?" }],
+    });
+    const result = await tools.get_weather?.execute?.(
+      { city: "SF" },
+      { toolCallId: "tc-1", messages: [] },
+    );
+    expect(executeTool).toHaveBeenCalledWith(
+      "get_weather",
+      { city: "SF" },
+      "sess-42",
+      [{ role: "user", content: "?" }],
+      { toolCallId: "tc-1" },
+    );
+    expect(result).toBe("rainy");
+  });
+
+  test("execute passes through abort signal when provided", async () => {
+    const controller = new AbortController();
+    const executeTool = vi.fn(
+      async (
+        _n: string,
+        _a: Readonly<Record<string, unknown>>,
+        _s?: string,
+        _m?: readonly unknown[],
+        opts?: { signal?: AbortSignal },
+      ) => {
+        expect(opts?.signal).toBe(controller.signal);
+        return "ok";
+      },
+    );
+    const tools = toVercelTools(schemas, {
+      executeTool,
+      sessionId: "s",
+      messages: () => [],
+      signal: controller.signal,
+    });
+    await tools.get_weather?.execute?.({ city: "NY" }, { toolCallId: "tc-2", messages: [] });
+    expect(executeTool).toHaveBeenCalledTimes(1);
+  });
+
+  test("execute prefers options.abortSignal over ctx.signal", async () => {
+    const ctxController = new AbortController();
+    const callController = new AbortController();
+    let receivedSignal: AbortSignal | undefined;
+    const executeTool = vi.fn(
+      async (
+        _n: string,
+        _a: Readonly<Record<string, unknown>>,
+        _s?: string,
+        _m?: readonly unknown[],
+        opts?: { signal?: AbortSignal },
+      ) => {
+        receivedSignal = opts?.signal;
+        return "ok";
+      },
+    );
+    const tools = toVercelTools(schemas, {
+      executeTool,
+      sessionId: "s",
+      messages: () => [],
+      signal: ctxController.signal,
+    });
+    await tools.get_weather?.execute?.(
+      { city: "NY" },
+      { toolCallId: "tc-1", messages: [], abortSignal: callController.signal },
+    );
+    expect(receivedSignal).toBe(callController.signal);
+  });
+
+  test("execute falls back to ctx.signal when options.abortSignal is absent", async () => {
+    const ctxController = new AbortController();
+    let receivedSignal: AbortSignal | undefined;
+    const executeTool = vi.fn(
+      async (
+        _n: string,
+        _a: Readonly<Record<string, unknown>>,
+        _s?: string,
+        _m?: readonly unknown[],
+        opts?: { signal?: AbortSignal },
+      ) => {
+        receivedSignal = opts?.signal;
+        return "ok";
+      },
+    );
+    const tools = toVercelTools(schemas, {
+      executeTool,
+      sessionId: "s",
+      messages: () => [],
+      signal: ctxController.signal,
+    });
+    await tools.get_weather?.execute?.({ city: "NY" }, { toolCallId: "tc-2", messages: [] });
+    expect(receivedSignal).toBe(ctxController.signal);
+  });
+
+  test("execute propagates toolCallId from options", async () => {
+    let receivedCallId: string | undefined;
+    const executeTool = vi.fn(
+      async (
+        _n: string,
+        _a: Readonly<Record<string, unknown>>,
+        _s?: string,
+        _m?: readonly unknown[],
+        opts?: { toolCallId?: string },
+      ) => {
+        receivedCallId = opts?.toolCallId;
+        return "ok";
+      },
+    );
+    const tools = toVercelTools(schemas, {
+      executeTool,
+      sessionId: "s",
+      messages: () => [],
+    });
+    await tools.get_weather?.execute?.({ city: "NY" }, { toolCallId: "tc-3", messages: [] });
+    expect(receivedCallId).toBe("tc-3");
+  });
+});

package/host/to-vercel-tools.ts

@@ -0,0 +1,70 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * Converts agent {@link ToolSchema}[] to Vercel AI SDK tools with `execute`
+ * delegation to the agent's {@link ExecuteTool} function.
+ *
+ * The pipeline orchestrator passes the output to `streamText({ tools })`.
+ * Each produced tool's `execute` closure calls
+ * `ctx.executeTool(name, args, sessionId, messages(), { signal, toolCallId })`,
+ * so the existing agent tool infrastructure (argument validation, KV, hooks,
+ * timeout) remains the single source of truth for tool behavior.
+ *
+ * Per-call `options.abortSignal` (forwarded by `streamText` when the
+ * outer turn is aborted, e.g. barge-in) takes precedence over the
+ * bag-level `ctx.signal` so individual invocations respect streamText
+ * aborts.
+ */
+
+import { jsonSchema, type Tool, type ToolExecutionOptions, tool } from "ai";
+import type { ExecuteTool, ExecuteToolOptions, ToolSchema } from "../sdk/_internal-types.ts";
+import type { Message } from "../sdk/types.ts";
+
+export interface ToVercelToolsContext {
+  /** The agent's tool-execution function (from the runtime). */
+  executeTool: ExecuteTool;
+  /** Session id threaded to {@link executeTool}. */
+  sessionId: string;
+  /**
+   * Returns the current conversation history at call-time. Called per
+   * tool invocation so late calls see fresh state instead of a snapshot
+   * captured when the tool bag was built.
+   */
+  messages: () => readonly Message[];
+  /**
+   * Bag-level abort signal. Used as a fallback when the per-call
+   * `options.abortSignal` from Vercel's `ToolExecutionOptions` is absent.
+   */
+  signal?: AbortSignal;
+}
+
+/**
+ * Convert an array of {@link ToolSchema} to a Vercel AI SDK `ToolSet`
+ * (record keyed by tool name).
+ *
+ * Uses the v6 `tool()` helper with `inputSchema: jsonSchema(...)` wrapping
+ * the agent's JSON Schema `parameters`. Execution is delegated to
+ * `ctx.executeTool` so validation, KV, timeouts, and hooks keep working.
+ */
+export function toVercelTools(
+  schemas: readonly ToolSchema[],
+  ctx: ToVercelToolsContext,
+): Record<string, Tool> {
+  const out: Record<string, Tool> = {};
+  for (const schema of schemas) {
+    out[schema.name] = tool({
+      description: schema.description,
+      inputSchema: jsonSchema(schema.parameters),
+      execute: async (args: unknown, options: ToolExecutionOptions) => {
+        const input = (args ?? {}) as Readonly<Record<string, unknown>>;
+        // Prefer the per-call abortSignal forwarded by streamText over the
+        // bag-level ctx.signal so individual invocations respect aborts.
+        const signal = options.abortSignal ?? ctx.signal;
+        const opts: ExecuteToolOptions = {};
+        if (signal !== undefined) opts.signal = signal;
+        if (options.toolCallId !== undefined) opts.toolCallId = options.toolCallId;
+        return ctx.executeTool(schema.name, input, ctx.sessionId, ctx.messages(), opts);
+      },
+    });
+  }
+  return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alexkroman1/aai",
-  "version": "1.2.3",
+  "version": "1.3.0",
   "type": "module",
   "exports": {
     ".": {
@@ -22,9 +22,22 @@
       "@dev/source": "./sdk/manifest-barrel.ts",
       "types": "./dist/sdk/manifest-barrel.d.ts",
       "import": "./dist/sdk/manifest-barrel.js"
+    },
+    "./stt": {
+      "@dev/source": "./host/providers/stt-barrel.ts",
+      "types": "./dist/host/providers/stt-barrel.d.ts",
+      "import": "./dist/host/providers/stt-barrel.js"
+    },
+    "./tts": {
+      "@dev/source": "./host/providers/tts-barrel.ts",
+      "types": "./dist/host/providers/tts-barrel.d.ts",
+      "import": "./dist/host/providers/tts-barrel.js"
     }
   },
   "dependencies": {
+    "@cartesia/cartesia-js": "^3.0.0",
+    "ai": "^6.0.161",
+    "assemblyai": "^4.30.0",
     "escape-html": "^1.0.3",
     "html-to-text": "^9.0.5",
     "mime-types": "^3.0.2",
@@ -35,6 +48,7 @@
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@ai-sdk/openai": "^3.0.0",
     "@types/escape-html": "^1.0.4",
     "@types/html-to-text": "^9.0.4",
     "@types/json-schema": "^7.0.15",

package/sdk/__snapshots__/exports.test.ts.snap

@@ -63,6 +63,7 @@ exports[`export surface stability > @alexkroman1/aai/protocol export 1`] = `
 exports[`export surface stability > @alexkroman1/aai/runtime export 1`] = `
 [
   "DEFAULT_S2S_CONFIG",
+  "_buildBaseCtx",
   "_internals",
   "buildCtx",
   "consoleLogger",

package/sdk/_internal-types.ts

@@ -10,6 +10,21 @@ import { z } from "zod";
 import type { Message } from "./types.ts";
 import { BuiltinToolSchema, ToolChoiceSchema, type ToolDef } from "./types.ts";
 
+/**
+ * Options forwarded to an {@link ExecuteTool} invocation.
+ *
+ * Primarily used by the pipeline orchestrator (streamText tool loop) to
+ * thread an {@link AbortSignal} into tool execution. The S2S voice path
+ * does not pass these options today — recipients must treat the whole
+ * bag as optional.
+ */
+export interface ExecuteToolOptions {
+  /** Abort signal bound to the enclosing LLM turn / request. */
+  signal?: AbortSignal;
+  /** Vercel AI SDK tool-call ID for this invocation. Useful for tracing and correlation. */
+  toolCallId?: string;
+}
+
 /**
  * Function signature for executing a tool by name.
  *
@@ -21,6 +36,7 @@ export type ExecuteTool = (
   args: Readonly<Record<string, unknown>>,
   sessionId?: string,
   messages?: readonly Message[],
+  opts?: ExecuteToolOptions,
 ) => Promise<string>;
 
 // ─── AgentConfig ────────────────────────────────────────────────────────────

package/sdk/define.test-d.ts

@@ -0,0 +1,21 @@
+// Copyright 2025 the AAI authors. MIT license.
+import { expectTypeOf, test } from "vitest";
+import { agent } from "./define.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
+
+test("agent() accepts stt/llm/tts optional fields", () => {
+  const stt = {} as SttProvider;
+  const llm = {} as LlmProvider;
+  const tts = {} as TtsProvider;
+  const def = agent({ name: "t", systemPrompt: "p", stt, llm, tts });
+  expectTypeOf(def.stt).toEqualTypeOf<SttProvider | undefined>();
+  expectTypeOf(def.llm).toEqualTypeOf<LlmProvider | undefined>();
+  expectTypeOf(def.tts).toEqualTypeOf<TtsProvider | undefined>();
+});
+
+test("agent() without stt/llm/tts is still legal (s2s mode)", () => {
+  const def = agent({ name: "t", systemPrompt: "p" });
+  expectTypeOf(def.stt).toEqualTypeOf<SttProvider | undefined>();
+  expectTypeOf(def.llm).toEqualTypeOf<LlmProvider | undefined>();
+  expectTypeOf(def.tts).toEqualTypeOf<TtsProvider | undefined>();
+});

package/sdk/define.test.ts

@@ -2,6 +2,8 @@
 import { describe, expect, test } from "vitest";
 import { z } from "zod";
 import { agent, tool } from "./define.ts";
+import { parseManifest } from "./manifest.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 
 describe("tool()", () => {
   test("returns the definition unchanged", () => {
@@ -54,4 +56,35 @@ describe("agent()", () => {
     expect(def.tools.greet).toBe(greetTool);
     expect(def.builtinTools).toEqual(["web_search"]);
   });
+
+  test("preserves stt/llm/tts providers on the returned def", () => {
+    const stt = { name: "fake-stt", open: async () => ({}) } as unknown as SttProvider;
+    const tts = { name: "fake-tts", open: async () => ({}) } as unknown as TtsProvider;
+    const llm = {} as LlmProvider;
+    const def = agent({ name: "t", systemPrompt: "p", stt, llm, tts });
+    expect(def.stt).toBe(stt);
+    expect(def.llm).toBe(llm);
+    expect(def.tts).toBe(tts);
+  });
+
+  test("stt/llm/tts flow through parseManifest to mode 'pipeline'", () => {
+    const stt = { name: "fake-stt", open: async () => ({}) } as unknown as SttProvider;
+    const tts = { name: "fake-tts", open: async () => ({}) } as unknown as TtsProvider;
+    const llm = {} as LlmProvider;
+    const def = agent({ name: "t", systemPrompt: "p", stt, llm, tts });
+    const parsed = parseManifest(def);
+    expect(parsed.mode).toBe("pipeline");
+    expect(parsed.stt).toBe(stt);
+    expect(parsed.llm).toBe(llm);
+    expect(parsed.tts).toBe(tts);
+  });
+
+  test("agent without providers resolves to mode 's2s'", () => {
+    const def = agent({ name: "t", systemPrompt: "p" });
+    const parsed = parseManifest(def);
+    expect(parsed.mode).toBe("s2s");
+    expect(parsed.stt).toBeUndefined();
+    expect(parsed.llm).toBeUndefined();
+    expect(parsed.tts).toBeUndefined();
+  });
 });

package/sdk/define.ts

@@ -4,6 +4,7 @@
  */
 
 import type { z } from "zod";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 import {
   type AgentDef,
   type BuiltinTool,
@@ -65,6 +66,11 @@ export function tool<P extends z.ZodObject<z.ZodRawShape>>(def: {
  * });
  * ```
  *
+ * @remarks
+ * Pipeline mode: pass `stt`, `llm`, and `tts` together to switch from the
+ * default AssemblyAI Streaming Speech-to-Speech path to a pluggable
+ * STT → LLM → TTS pipeline. All three must be set (or all left unset).
+ *
  * @public
  */
 export function agent(def: {
@@ -77,6 +83,21 @@ export function agent(def: {
   toolChoice?: ToolChoice;
   sttPrompt?: string;
   idleTimeoutMs?: number;
+  /**
+   * Pluggable STT provider. Must be set together with `llm` and `tts` to
+   * enable pipeline mode; leave all three unset for S2S mode.
+   */
+  stt?: SttProvider;
+  /**
+   * Pluggable LLM provider (Vercel AI SDK `LanguageModel`). Must be set
+   * together with `stt` and `tts` to enable pipeline mode.
+   */
+  llm?: LlmProvider;
+  /**
+   * Pluggable TTS provider. Must be set together with `stt` and `llm` to
+   * enable pipeline mode.
+   */
+  tts?: TtsProvider;
 }): AgentDef {
   return {
     systemPrompt: DEFAULT_SYSTEM_PROMPT,

package/sdk/manifest.test-d.ts

@@ -0,0 +1,14 @@
+// Copyright 2025 the AAI authors. MIT license.
+import { expectTypeOf, test } from "vitest";
+import type { Manifest } from "./manifest.ts";
+
+test("Manifest.stt/llm/tts are optional", () => {
+  expectTypeOf<Manifest["stt"]>().toBeNullable();
+  expectTypeOf<Manifest["llm"]>().toBeNullable();
+  expectTypeOf<Manifest["tts"]>().toBeNullable();
+});
+
+test("parseManifest return includes mode", () => {
+  type Parsed = ReturnType<typeof import("./manifest.ts").parseManifest>;
+  expectTypeOf<Parsed["mode"]>().toEqualTypeOf<"s2s" | "pipeline">();
+});