@alexkroman1/aai 1.2.3 → 1.3.1

package/sdk/constants.ts

@@ -34,6 +34,14 @@ export const RUN_CODE_TIMEOUT_MS = 5000;
 /** Maximum time to wait for sessions to stop during graceful shutdown. */
 export const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
 
+/**
+ * Maximum time to wait for a pipeline-mode TTS drain after `flush()` before
+ * forcing the turn to complete. Prevents a stuck TTS provider from wedging
+ * the session. Short relative to `DEFAULT_SHUTDOWN_TIMEOUT_MS` so stop()
+ * can still reclaim the socket cleanly.
+ */
+export const PIPELINE_FLUSH_TIMEOUT_MS = 10_000;
+
 // ─── Size / length limits ────────────────────────────────────────────────
 
 /** Maximum length for tool result strings sent to clients. */

package/sdk/define.test-d.ts

@@ -0,0 +1,21 @@
+// Copyright 2025 the AAI authors. MIT license.
+import { expectTypeOf, test } from "vitest";
+import { agent } from "./define.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
+
+test("agent() accepts stt/llm/tts optional fields", () => {
+  const stt = {} as SttProvider;
+  const llm = {} as LlmProvider;
+  const tts = {} as TtsProvider;
+  const def = agent({ name: "t", systemPrompt: "p", stt, llm, tts });
+  expectTypeOf(def.stt).toEqualTypeOf<SttProvider | undefined>();
+  expectTypeOf(def.llm).toEqualTypeOf<LlmProvider | undefined>();
+  expectTypeOf(def.tts).toEqualTypeOf<TtsProvider | undefined>();
+});
+
+test("agent() without stt/llm/tts is still legal (s2s mode)", () => {
+  const def = agent({ name: "t", systemPrompt: "p" });
+  expectTypeOf(def.stt).toEqualTypeOf<SttProvider | undefined>();
+  expectTypeOf(def.llm).toEqualTypeOf<LlmProvider | undefined>();
+  expectTypeOf(def.tts).toEqualTypeOf<TtsProvider | undefined>();
+});

package/sdk/define.test.ts

@@ -2,6 +2,8 @@
 import { describe, expect, test } from "vitest";
 import { z } from "zod";
 import { agent, tool } from "./define.ts";
+import { parseManifest } from "./manifest.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 
 describe("tool()", () => {
   test("returns the definition unchanged", () => {
@@ -54,4 +56,35 @@ describe("agent()", () => {
     expect(def.tools.greet).toBe(greetTool);
     expect(def.builtinTools).toEqual(["web_search"]);
   });
+
+  test("preserves stt/llm/tts providers on the returned def", () => {
+    const stt = { name: "fake-stt", open: async () => ({}) } as unknown as SttProvider;
+    const tts = { name: "fake-tts", open: async () => ({}) } as unknown as TtsProvider;
+    const llm = {} as LlmProvider;
+    const def = agent({ name: "t", systemPrompt: "p", stt, llm, tts });
+    expect(def.stt).toBe(stt);
+    expect(def.llm).toBe(llm);
+    expect(def.tts).toBe(tts);
+  });
+
+  test("stt/llm/tts flow through parseManifest to mode 'pipeline'", () => {
+    const stt = { name: "fake-stt", open: async () => ({}) } as unknown as SttProvider;
+    const tts = { name: "fake-tts", open: async () => ({}) } as unknown as TtsProvider;
+    const llm = {} as LlmProvider;
+    const def = agent({ name: "t", systemPrompt: "p", stt, llm, tts });
+    const parsed = parseManifest(def);
+    expect(parsed.mode).toBe("pipeline");
+    expect(parsed.stt).toBe(stt);
+    expect(parsed.llm).toBe(llm);
+    expect(parsed.tts).toBe(tts);
+  });
+
+  test("agent without providers resolves to mode 's2s'", () => {
+    const def = agent({ name: "t", systemPrompt: "p" });
+    const parsed = parseManifest(def);
+    expect(parsed.mode).toBe("s2s");
+    expect(parsed.stt).toBeUndefined();
+    expect(parsed.llm).toBeUndefined();
+    expect(parsed.tts).toBeUndefined();
+  });
 });

package/sdk/define.ts

@@ -4,6 +4,7 @@
  */
 
 import type { z } from "zod";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 import {
   type AgentDef,
   type BuiltinTool,
@@ -65,6 +66,11 @@ export function tool<P extends z.ZodObject<z.ZodRawShape>>(def: {
  * });
  * ```
  *
+ * @remarks
+ * Pipeline mode: pass `stt`, `llm`, and `tts` together to switch from the
+ * default AssemblyAI Streaming Speech-to-Speech path to a pluggable
+ * STT → LLM → TTS pipeline. All three must be set (or all left unset).
+ *
  * @public
  */
 export function agent(def: {
@@ -77,6 +83,21 @@ export function agent(def: {
   toolChoice?: ToolChoice;
   sttPrompt?: string;
   idleTimeoutMs?: number;
+  /**
+   * Pluggable STT provider. Must be set together with `llm` and `tts` to
+   * enable pipeline mode; leave all three unset for S2S mode.
+   */
+  stt?: SttProvider;
+  /**
+   * Pluggable LLM provider (Vercel AI SDK `LanguageModel`). Must be set
+   * together with `stt` and `tts` to enable pipeline mode.
+   */
+  llm?: LlmProvider;
+  /**
+   * Pluggable TTS provider. Must be set together with `stt` and `llm` to
+   * enable pipeline mode.
+   */
+  tts?: TtsProvider;
 }): AgentDef {
   return {
     systemPrompt: DEFAULT_SYSTEM_PROMPT,

package/sdk/manifest.test-d.ts

@@ -0,0 +1,14 @@
+// Copyright 2025 the AAI authors. MIT license.
+import { expectTypeOf, test } from "vitest";
+import type { Manifest } from "./manifest.ts";
+
+test("Manifest.stt/llm/tts are optional", () => {
+  expectTypeOf<Manifest["stt"]>().toBeNullable();
+  expectTypeOf<Manifest["llm"]>().toBeNullable();
+  expectTypeOf<Manifest["tts"]>().toBeNullable();
+});
+
+test("parseManifest return includes mode", () => {
+  type Parsed = ReturnType<typeof import("./manifest.ts").parseManifest>;
+  expectTypeOf<Parsed["mode"]>().toEqualTypeOf<"s2s" | "pipeline">();
+});

package/sdk/manifest.test.ts

@@ -12,11 +12,18 @@ describe("parseManifest", () => {
       name: "Simple Agent",
       systemPrompt: expect.any(String),
       greeting: expect.any(String),
+      sttPrompt: undefined,
       maxSteps: 5,
       toolChoice: "auto",
+      idleTimeoutMs: undefined,
+      theme: undefined,
       builtinTools: [],
       allowedHosts: [],
       tools: {},
+      stt: undefined,
+      llm: undefined,
+      tts: undefined,
+      mode: "s2s",
     });
   });
 
@@ -155,3 +162,47 @@ describe("manifest type contracts", () => {
     expectTypeOf(result.allowedHosts).toEqualTypeOf<string[]>();
   });
 });
+
+describe("parseManifest — mode classification", () => {
+  const stubStt = { name: "stub-stt", open: async () => ({}) };
+  const stubTts = { name: "stub-tts", open: async () => ({}) };
+  const stubLlm = { modelId: "stub-llm" };
+
+  test("no stt/llm/tts ⇒ mode: 's2s'", () => {
+    const parsed = parseManifest({
+      name: "hello",
+      systemPrompt: "hi",
+    });
+    expect(parsed.mode).toBe("s2s");
+  });
+
+  test("all three of stt/llm/tts set ⇒ mode: 'pipeline'", () => {
+    const parsed = parseManifest({
+      name: "hello",
+      systemPrompt: "hi",
+      stt: stubStt,
+      llm: stubLlm,
+      tts: stubTts,
+    } as never);
+    expect(parsed.mode).toBe("pipeline");
+  });
+
+  test("only stt set ⇒ throws", () => {
+    expect(() =>
+      parseManifest({
+        name: "hello",
+        stt: stubStt,
+      } as never),
+    ).toThrow(/stt, llm, and tts must be set together/);
+  });
+
+  test("stt + tts without llm ⇒ throws", () => {
+    expect(() =>
+      parseManifest({
+        name: "hello",
+        stt: stubStt,
+        tts: stubTts,
+      } as never),
+    ).toThrow(/stt, llm, and tts must be set together/);
+  });
+});

package/sdk/manifest.ts

@@ -8,6 +8,7 @@
 
 import { z } from "zod";
 import { validateAllowedHostPattern } from "./allowed-hosts.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 import { BuiltinToolSchema, DEFAULT_GREETING, DEFAULT_SYSTEM_PROMPT } from "./types.ts";
 
 /**
@@ -46,6 +47,27 @@ export type Manifest = {
   tools: Record<string, ToolManifest>;
   /** Hostnames the agent is allowed to fetch. Empty = no fetch access. */
   allowedHosts: string[];
+  /**
+   * Pluggable STT provider. Must be set together with `llm` and `tts` to
+   * enable pipeline mode, or all three left unset for s2s mode.
+   */
+  stt?: SttProvider | undefined;
+  /**
+   * Pluggable LLM provider (Vercel AI SDK `LanguageModel`). Must be set
+   * together with `stt` and `tts` to enable pipeline mode.
+   */
+  llm?: LlmProvider | undefined;
+  /**
+   * Pluggable TTS provider. Must be set together with `stt` and `llm` to
+   * enable pipeline mode.
+   */
+  tts?: TtsProvider | undefined;
+  /**
+   * Session mode derived from provider fields:
+   * - `"s2s"` (default): AssemblyAI Streaming Speech-to-Speech path (no stt/llm/tts set).
+   * - `"pipeline"`: pluggable STT → LLM → TTS path (stt + llm + tts all set).
+   */
+  mode: "s2s" | "pipeline";
 };
 
 const ToolManifestSchema = z.object({
@@ -92,6 +114,19 @@ const ManifestSchema = z.object({
  */
 export function parseManifest(input: unknown): Manifest {
   const parsed = ManifestSchema.parse(input);
+  // stt/llm/tts are runtime objects (functions/classes) that can't be
+  // validated by the JSON-oriented Zod schema. Pull them straight from
+  // the raw input and enforce all-or-nothing presence here.
+  const raw = (input ?? {}) as {
+    stt?: SttProvider;
+    llm?: LlmProvider;
+    tts?: TtsProvider;
+  };
+  const providerCount = [raw.stt, raw.llm, raw.tts].filter((x) => x != null).length;
+  if (providerCount !== 0 && providerCount !== 3) {
+    throw new Error("stt, llm, and tts must be set together");
+  }
+  const mode: "s2s" | "pipeline" = providerCount === 3 ? "pipeline" : "s2s";
   return {
     name: parsed.name,
     systemPrompt: parsed.systemPrompt ?? DEFAULT_SYSTEM_PROMPT,
@@ -104,5 +139,9 @@ export function parseManifest(input: unknown): Manifest {
     theme: parsed.theme,
     tools: parsed.tools ?? {},
     allowedHosts: parsed.allowedHosts ?? [],
+    stt: raw.stt,
+    llm: raw.llm,
+    tts: raw.tts,
+    mode,
   };
 }

package/sdk/providers.ts

@@ -0,0 +1,90 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * Pluggable provider interfaces — normalized seams over streaming STT / TTS
+ * SDKs, plus the LLM provider type.
+ *
+ * These are zero-runtime **type** declarations with no Node.js dependencies,
+ * so they live in `sdk/` alongside the `Manifest` type that references them.
+ * Concrete adapters (e.g. AssemblyAI STT, Cartesia TTS) live under
+ * `host/providers/` because they depend on Node-only SDKs.
+ */
+
+import type { LanguageModel } from "ai";
+
+/** Unsubscribe callback returned by `.on()` event subscriptions. */
+export type Unsubscribe = () => void;
+
+// -------- STT --------
+
+export interface SttError extends Error {
+  readonly code: "stt_connect_failed" | "stt_auth_failed" | "stt_stream_error";
+}
+
+export type SttEvents = {
+  /** Interim transcript; drives barge-in detection. */
+  partial: (text: string) => void;
+  /** End-of-turn final transcript; cue to run the LLM. */
+  final: (text: string) => void;
+  /** Terminal error. The session is expected to end after this fires. */
+  error: (err: SttError) => void;
+};
+
+export interface SttSession {
+  sendAudio(pcm: Int16Array): void;
+  on<E extends keyof SttEvents>(event: E, fn: SttEvents[E]): Unsubscribe;
+  close(): Promise<void>;
+}
+
+export interface SttOpenOptions {
+  sampleRate: number;
+  apiKey: string;
+  sttPrompt?: string | undefined;
+  signal: AbortSignal;
+}
+
+export interface SttProvider {
+  readonly name: string;
+  open(opts: SttOpenOptions): Promise<SttSession>;
+}
+
+// -------- TTS --------
+
+export interface TtsError extends Error {
+  readonly code: "tts_connect_failed" | "tts_auth_failed" | "tts_stream_error";
+}
+
+export type TtsEvents = {
+  /** One PCM16 audio chunk. Orchestrator forwards to the client. */
+  audio: (pcm: Int16Array) => void;
+  /** Synthesis drained after flush() or cancel(). Emitted exactly once per turn. */
+  done: () => void;
+  /** Terminal error. The session is expected to end after this fires. */
+  error: (err: TtsError) => void;
+};
+
+export interface TtsSession {
+  /** Push text deltas from the LLM. Provider may synthesize as chunks arrive. */
+  sendText(text: string): void;
+  /** Signal "no more text this turn". Emits `done` when fully synthesized. */
+  flush(): void;
+  /** Interrupt immediately (barge-in). Emits `done` synchronously. */
+  cancel(): void;
+  on<E extends keyof TtsEvents>(event: E, fn: TtsEvents[E]): Unsubscribe;
+  close(): Promise<void>;
+}
+
+export interface TtsOpenOptions {
+  sampleRate: number;
+  apiKey: string;
+  signal: AbortSignal;
+}
+
+export interface TtsProvider {
+  readonly name: string;
+  open(opts: TtsOpenOptions): Promise<TtsSession>;
+}
+
+// -------- LLM --------
+
+/** LLM provider — Vercel AI SDK's `LanguageModel`; no wrapping. */
+export type LlmProvider = LanguageModel;

package/sdk/types.ts

@@ -5,6 +5,7 @@
 
 import { z } from "zod";
 import type { Kv } from "./kv.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 
 /**
  * Identifier for a built-in server-side tool.
@@ -217,6 +218,21 @@ export type AgentDef<S = Record<string, unknown>> = {
   tools: Readonly<Record<string, ToolDef<z.ZodObject<z.ZodRawShape>, S>>>;
   state?: () => S;
   idleTimeoutMs?: number;
+  /**
+   * Pluggable STT provider. Set together with `llm` and `tts` to enable
+   * pipeline mode; all three unset means S2S mode.
+   */
+  stt?: SttProvider;
+  /**
+   * Pluggable LLM provider (Vercel AI SDK `LanguageModel`). Set together
+   * with `stt` and `tts` for pipeline mode.
+   */
+  llm?: LlmProvider;
+  /**
+   * Pluggable TTS provider. Set together with `stt` and `llm` for
+   * pipeline mode.
+   */
+  tts?: TtsProvider;
 };
 
 // ─── Zod schemas ────────────────────────────────────────────────────────────

package/vitest.config.ts

@@ -10,6 +10,7 @@ export default defineConfig({
       "**/pentest.test.ts",
       "**/run-code-sandbox.test.ts",
       "**/integration.test.ts",
+      "**/*.integration.test.ts",
       "node_modules",
       "dist",
     ],