@alexkroman1/aai 1.2.3 → 1.3.0

package/sdk/manifest.test.ts

@@ -12,11 +12,18 @@ describe("parseManifest", () => {
       name: "Simple Agent",
       systemPrompt: expect.any(String),
       greeting: expect.any(String),
+      sttPrompt: undefined,
       maxSteps: 5,
       toolChoice: "auto",
+      idleTimeoutMs: undefined,
+      theme: undefined,
       builtinTools: [],
       allowedHosts: [],
       tools: {},
+      stt: undefined,
+      llm: undefined,
+      tts: undefined,
+      mode: "s2s",
     });
   });
 
@@ -155,3 +162,47 @@ describe("manifest type contracts", () => {
     expectTypeOf(result.allowedHosts).toEqualTypeOf<string[]>();
   });
 });
+
+describe("parseManifest — mode classification", () => {
+  const stubStt = { name: "stub-stt", open: async () => ({}) };
+  const stubTts = { name: "stub-tts", open: async () => ({}) };
+  const stubLlm = { modelId: "stub-llm" };
+
+  test("no stt/llm/tts ⇒ mode: 's2s'", () => {
+    const parsed = parseManifest({
+      name: "hello",
+      systemPrompt: "hi",
+    });
+    expect(parsed.mode).toBe("s2s");
+  });
+
+  test("all three of stt/llm/tts set ⇒ mode: 'pipeline'", () => {
+    const parsed = parseManifest({
+      name: "hello",
+      systemPrompt: "hi",
+      stt: stubStt,
+      llm: stubLlm,
+      tts: stubTts,
+    } as never);
+    expect(parsed.mode).toBe("pipeline");
+  });
+
+  test("only stt set ⇒ throws", () => {
+    expect(() =>
+      parseManifest({
+        name: "hello",
+        stt: stubStt,
+      } as never),
+    ).toThrow(/stt, llm, and tts must be set together/);
+  });
+
+  test("stt + tts without llm ⇒ throws", () => {
+    expect(() =>
+      parseManifest({
+        name: "hello",
+        stt: stubStt,
+        tts: stubTts,
+      } as never),
+    ).toThrow(/stt, llm, and tts must be set together/);
+  });
+});

package/sdk/manifest.ts

@@ -8,6 +8,7 @@
 
 import { z } from "zod";
 import { validateAllowedHostPattern } from "./allowed-hosts.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 import { BuiltinToolSchema, DEFAULT_GREETING, DEFAULT_SYSTEM_PROMPT } from "./types.ts";
 
 /**
@@ -46,6 +47,27 @@ export type Manifest = {
   tools: Record<string, ToolManifest>;
   /** Hostnames the agent is allowed to fetch. Empty = no fetch access. */
   allowedHosts: string[];
+  /**
+   * Pluggable STT provider. Must be set together with `llm` and `tts` to
+   * enable pipeline mode, or all three left unset for s2s mode.
+   */
+  stt?: SttProvider | undefined;
+  /**
+   * Pluggable LLM provider (Vercel AI SDK `LanguageModel`). Must be set
+   * together with `stt` and `tts` to enable pipeline mode.
+   */
+  llm?: LlmProvider | undefined;
+  /**
+   * Pluggable TTS provider. Must be set together with `stt` and `llm` to
+   * enable pipeline mode.
+   */
+  tts?: TtsProvider | undefined;
+  /**
+   * Session mode derived from provider fields:
+   * - `"s2s"` (default): AssemblyAI Streaming Speech-to-Speech path (no stt/llm/tts set).
+   * - `"pipeline"`: pluggable STT → LLM → TTS path (stt + llm + tts all set).
+   */
+  mode: "s2s" | "pipeline";
 };
 
 const ToolManifestSchema = z.object({
@@ -92,6 +114,19 @@ const ManifestSchema = z.object({
  */
 export function parseManifest(input: unknown): Manifest {
   const parsed = ManifestSchema.parse(input);
+  // stt/llm/tts are runtime objects (functions/classes) that can't be
+  // validated by the JSON-oriented Zod schema. Pull them straight from
+  // the raw input and enforce all-or-nothing presence here.
+  const raw = (input ?? {}) as {
+    stt?: SttProvider;
+    llm?: LlmProvider;
+    tts?: TtsProvider;
+  };
+  const providerCount = [raw.stt, raw.llm, raw.tts].filter((x) => x != null).length;
+  if (providerCount !== 0 && providerCount !== 3) {
+    throw new Error("stt, llm, and tts must be set together");
+  }
+  const mode: "s2s" | "pipeline" = providerCount === 3 ? "pipeline" : "s2s";
   return {
     name: parsed.name,
     systemPrompt: parsed.systemPrompt ?? DEFAULT_SYSTEM_PROMPT,
@@ -104,5 +139,9 @@ export function parseManifest(input: unknown): Manifest {
     theme: parsed.theme,
     tools: parsed.tools ?? {},
     allowedHosts: parsed.allowedHosts ?? [],
+    stt: raw.stt,
+    llm: raw.llm,
+    tts: raw.tts,
+    mode,
   };
 }

package/sdk/providers.ts

@@ -0,0 +1,90 @@
+// Copyright 2025 the AAI authors. MIT license.
+/**
+ * Pluggable provider interfaces — normalized seams over streaming STT / TTS
+ * SDKs, plus the LLM provider type.
+ *
+ * These are zero-runtime **type** declarations with no Node.js dependencies,
+ * so they live in `sdk/` alongside the `Manifest` type that references them.
+ * Concrete adapters (e.g. AssemblyAI STT, Cartesia TTS) live under
+ * `host/providers/` because they depend on Node-only SDKs.
+ */
+
+import type { LanguageModel } from "ai";
+
+/** Unsubscribe callback returned by `.on()` event subscriptions. */
+export type Unsubscribe = () => void;
+
+// -------- STT --------
+
+export interface SttError extends Error {
+  readonly code: "stt_connect_failed" | "stt_auth_failed" | "stt_stream_error";
+}
+
+export type SttEvents = {
+  /** Interim transcript; drives barge-in detection. */
+  partial: (text: string) => void;
+  /** End-of-turn final transcript; cue to run the LLM. */
+  final: (text: string) => void;
+  /** Terminal error. The session is expected to end after this fires. */
+  error: (err: SttError) => void;
+};
+
+export interface SttSession {
+  sendAudio(pcm: Int16Array): void;
+  on<E extends keyof SttEvents>(event: E, fn: SttEvents[E]): Unsubscribe;
+  close(): Promise<void>;
+}
+
+export interface SttOpenOptions {
+  sampleRate: number;
+  apiKey: string;
+  sttPrompt?: string | undefined;
+  signal: AbortSignal;
+}
+
+export interface SttProvider {
+  readonly name: string;
+  open(opts: SttOpenOptions): Promise<SttSession>;
+}
+
+// -------- TTS --------
+
+export interface TtsError extends Error {
+  readonly code: "tts_connect_failed" | "tts_auth_failed" | "tts_stream_error";
+}
+
+export type TtsEvents = {
+  /** One PCM16 audio chunk. Orchestrator forwards to the client. */
+  audio: (pcm: Int16Array) => void;
+  /** Synthesis drained after flush() or cancel(). Emitted exactly once per turn. */
+  done: () => void;
+  /** Terminal error. The session is expected to end after this fires. */
+  error: (err: TtsError) => void;
+};
+
+export interface TtsSession {
+  /** Push text deltas from the LLM. Provider may synthesize as chunks arrive. */
+  sendText(text: string): void;
+  /** Signal "no more text this turn". Emits `done` when fully synthesized. */
+  flush(): void;
+  /** Interrupt immediately (barge-in). Emits `done` synchronously. */
+  cancel(): void;
+  on<E extends keyof TtsEvents>(event: E, fn: TtsEvents[E]): Unsubscribe;
+  close(): Promise<void>;
+}
+
+export interface TtsOpenOptions {
+  sampleRate: number;
+  apiKey: string;
+  signal: AbortSignal;
+}
+
+export interface TtsProvider {
+  readonly name: string;
+  open(opts: TtsOpenOptions): Promise<TtsSession>;
+}
+
+// -------- LLM --------
+
+/** LLM provider — Vercel AI SDK's `LanguageModel`; no wrapping. */
+export type LlmProvider = LanguageModel;

package/sdk/types.ts

@@ -5,6 +5,7 @@
 
 import { z } from "zod";
 import type { Kv } from "./kv.ts";
+import type { LlmProvider, SttProvider, TtsProvider } from "./providers.ts";
 
 /**
  * Identifier for a built-in server-side tool.
@@ -217,6 +218,21 @@ export type AgentDef<S = Record<string, unknown>> = {
   tools: Readonly<Record<string, ToolDef<z.ZodObject<z.ZodRawShape>, S>>>;
   state?: () => S;
   idleTimeoutMs?: number;
+  /**
+   * Pluggable STT provider. Set together with `llm` and `tts` to enable
+   * pipeline mode; all three unset means S2S mode.
+   */
+  stt?: SttProvider;
+  /**
+   * Pluggable LLM provider (Vercel AI SDK `LanguageModel`). Set together
+   * with `stt` and `tts` for pipeline mode.
+   */
+  llm?: LlmProvider;
+  /**
+   * Pluggable TTS provider. Set together with `stt` and `llm` for
+   * pipeline mode.
+   */
+  tts?: TtsProvider;
 };
 
 // ─── Zod schemas ────────────────────────────────────────────────────────────

package/vitest.config.ts

@@ -10,6 +10,7 @@ export default defineConfig({
       "**/pentest.test.ts",
       "**/run-code-sandbox.test.ts",
       "**/integration.test.ts",
+      "**/*.integration.test.ts",
       "node_modules",
       "dist",
     ],