@alexkroman1/aai 1.4.5 → 1.5.0

package/host/providers/tts/rime.ts

@@ -0,0 +1,322 @@
+// Copyright 2026 the AAI authors. MIT license.
+/**
+ * Rime TTS opener (host-only).
+ *
+ * The user-facing descriptor factory (`rime(...)`) lives in
+ * `sdk/providers/tts/rime.ts`. This module is the host-side
+ * counterpart: it takes the descriptor options + an API key and
+ * returns a {@link TtsOpener} that the pipeline session drives.
+ *
+ * **Protocol.** Connects to Rime's `ws2` JSON WebSocket endpoint
+ * (`wss://users-ws.rime.ai/ws2`). Client-to-server messages are JSON:
+ *   - `{ "text": "..." }` — append text to the synthesis buffer
+ *   - `{ "operation": "clear" }` — drop buffered text (barge-in)
+ *   - `{ "operation": "eos" }` — drain buffer, close connection (NOT used
+ *     during a session: it would tear down the WS, forcing reconnect per
+ *     turn). We force end-of-turn synthesis with a trailing `"."` instead.
+ * The server responds with JSON frames:
+ *   - `{ type: "chunk", data: <base64 PCM16 LE>, contextId: string | null }`
+ *   - `{ type: "timestamps", ... }` (ignored)
+ *   - `{ type: "error", message: string }` (surfaced as `tts_stream_error`)
+ *
+ * **Single long-lived connection per session.** Rime buffers text until it
+ * sees terminal punctuation (`.`, `?`, `!`), so we use one WebSocket per
+ * `open()` call and reuse it across turns. `clear` resets the buffer
+ * between cancellations.
+ *
+ * **Done detection.** After `flush()` sends a trailing `"."` to force the
+ * server to synthesize any half-buffered text, we arm a quiescence timer
+ * that fires 500 ms after the last received audio chunk. When it fires,
+ * `done` is emitted.
+ *
+ * **Audio format.** The URL requests `audioFormat=pcm` at the negotiated
+ * `sampleRate`, which returns raw PCM16 little-endian. We decode the base64
+ * payload and construct a zero-copy `Int16Array` view over the decoded bytes.
+ */
+
+import { createNanoEvents, type Emitter } from "nanoevents";
+import WebSocket from "ws";
+import { RIME_DEFAULT_VOICE, type RimeOptions } from "../../../sdk/providers/tts/rime.ts";
+import {
+  makeTtsError,
+  type TtsEvents,
+  type TtsOpener,
+  type TtsOpenOptions,
+  type TtsSession,
+} from "../../../sdk/providers.ts";
+
+/** Internal: TtsSession with a test-only handle to the raw WebSocket. */
+export interface RimeSession extends TtsSession {
+  /** @internal Test-only: exposes the underlying raw WebSocket. */
+  readonly _ws: WebSocket;
+}
+
+/** PCM16 sample rates accepted by the Rime `ws2` endpoint. */
+const RIME_PCM16_RATES = [
+  8000, 16_000, 22_050, 24_000, 44_100, 48_000,
+] as const satisfies readonly number[];
+
+function assertSupportedSampleRate(rate: number): number {
+  if ((RIME_PCM16_RATES as readonly number[]).includes(rate)) return rate;
+  throw makeTtsError(
+    "tts_connect_failed",
+    `Rime TTS: unsupported sample rate ${rate}. Supported: ${RIME_PCM16_RATES.join(", ")}.`,
+  );
+}
+
+/**
+ * Decode a base64 string from Rime into a zero-copy `Int16Array`.
+ *
+ * Rime's `ws2` endpoint returns base64-encoded PCM16 LE in each chunk.
+ * `Buffer.from(base64, "base64")` gives us a Node.js Buffer (which is a
+ * Uint8Array subclass) with `byteOffset === 0`. PCM16 bytes always come in
+ * pairs so the length is guaranteed to be even.
+ */
+function base64ToPcm(data: string): Int16Array {
+  const bytes = Buffer.from(data, "base64");
+  // Defensive: drop a trailing odd byte rather than throwing on misalignment.
+  const evenLen = bytes.byteLength - (bytes.byteLength % 2);
+  if (evenLen === 0) return new Int16Array(0);
+  return new Int16Array(bytes.buffer, bytes.byteOffset, evenLen / 2);
+}
+
+/**
+ * Shape of JSON messages received from Rime's `ws2` endpoint.
+ *
+ * Only `chunk` messages carry audio; `timestamps` messages are informational
+ * and can be ignored for the current use case.
+ */
+interface RimeMessage {
+  type: "chunk" | "timestamps" | "error" | string;
+  /** Base64-encoded PCM16 LE audio. Present on `type === "chunk"`. */
+  data?: string;
+  /** Context discriminator for the in-flight utterance. May be null. */
+  contextId?: string | null;
+  /** Error description. Present on `type === "error"`. */
+  message?: string;
+}
+
+/** Quiescence timeout in ms — how long to wait after the last audio chunk before emitting `done`. */
+const QUIESCENCE_MS = 500;
+
+/**
+ * After `flush()`, how long to wait for the FIRST audio chunk before
+ * giving up and emitting `done`. Greeting and short replies hit this
+ * path: `flush()` runs immediately after `sendText()`, so audio TTFB
+ * exceeds the 500 ms quiescence window. Once the first chunk arrives,
+ * we transition to the shorter quiescence timeout.
+ */
+const FIRST_AUDIO_TIMEOUT_MS = 5000;
+
+/** Wait for the WebSocket `open` event; reject on first `error`. */
+function waitForOpen(ws: WebSocket): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    const onOpen = () => {
+      ws.removeListener("error", onError);
+      resolve();
+    };
+    const onError = (err: Error) => {
+      ws.removeListener("open", onOpen);
+      reject(
+        makeTtsError(
+          "tts_connect_failed",
+          `Rime TTS: connect failed: ${err?.message ?? String(err)}`,
+        ),
+      );
+    };
+    ws.once("open", onOpen);
+    ws.once("error", onError);
+  });
+}
+
+/**
+ * Handle one incoming WebSocket message frame.
+ *
+ * Extracted into a top-level function to keep `open()` under the cognitive
+ * complexity limit while retaining full access to the session state via refs.
+ */
+function handleRimeMessage(
+  raw: WebSocket.Data,
+  emitter: Emitter<TtsEvents>,
+  armQuiescence: () => void,
+  isActiveTimer: () => boolean,
+): void {
+  let msg: RimeMessage;
+  try {
+    msg = JSON.parse(typeof raw === "string" ? raw : raw.toString()) as RimeMessage;
+  } catch {
+    // Unparseable frame — ignore.
+    return;
+  }
+
+  if (msg.type === "chunk" && typeof msg.data === "string") {
+    const pcm = base64ToPcm(msg.data);
+    if (pcm.length > 0) {
+      emitter.emit("audio", pcm);
+      // While we're waiting on a flush (long timer for first audio, or
+      // short timer between chunks), each chunk resets to the short
+      // quiescence window — so `done` fires only after audio stops.
+      if (isActiveTimer()) armQuiescence();
+    }
+    return;
+  }
+  if (msg.type === "error") {
+    emitter.emit(
+      "error",
+      makeTtsError("tts_stream_error", `Rime TTS: ${msg.message ?? "unknown error"}`),
+    );
+  }
+  // Ignore `type === "timestamps"` and unknown message types.
+}
+
+/** Build a {@link TtsOpener} from resolved Rime descriptor options. */
+export function openRime(opts: RimeOptions): TtsOpener {
+  return {
+    name: "rime",
+    async open(openOpts: TtsOpenOptions): Promise<TtsSession> {
+      const apiKey = openOpts.apiKey || process.env.RIME_API_KEY;
+      if (!apiKey) {
+        throw makeTtsError(
+          "tts_auth_failed",
+          "Rime TTS: missing API key. Set RIME_API_KEY in the agent env.",
+        );
+      }
+
+      const sampleRate = assertSupportedSampleRate(openOpts.sampleRate);
+      const model = opts.model ?? "mistv2";
+      const lang = opts.language ?? "eng";
+      const voice = opts.voice ?? RIME_DEFAULT_VOICE;
+
+      // Construct the ws2 URL with query parameters.
+      const url = `wss://users-ws.rime.ai/ws2?speaker=${encodeURIComponent(voice)}&modelId=${encodeURIComponent(model)}&audioFormat=pcm&samplingRate=${sampleRate}&lang=${encodeURIComponent(lang)}`;
+
+      let ws: WebSocket;
+      try {
+        ws = new WebSocket(url, { headers: { Authorization: `Bearer ${apiKey}` } });
+      } catch (cause) {
+        throw makeTtsError(
+          "tts_connect_failed",
+          `Rime TTS: failed to create WebSocket: ${cause instanceof Error ? cause.message : String(cause)}`,
+        );
+      }
+
+      await waitForOpen(ws);
+
+      const emitter: Emitter<TtsEvents> = createNanoEvents<TtsEvents>();
+      let closed = false;
+      let doneEmitted = false;
+      /**
+       * After `flush()`, we arm a timer that fires `done`. Initial timeout is
+       * `FIRST_AUDIO_TIMEOUT_MS` to give Rime headroom on TTFB; the first
+       * chunk swaps it for a shorter `QUIESCENCE_MS` window that resets on
+       * each subsequent chunk. `cancel()` emits `done` synchronously.
+       */
+      let quiescenceTimer: ReturnType<typeof setTimeout> | null = null;
+
+      const clearQuiescence = () => {
+        if (quiescenceTimer !== null) {
+          clearTimeout(quiescenceTimer);
+          quiescenceTimer = null;
+        }
+      };
+
+      const emitDoneOnce = () => {
+        clearQuiescence();
+        if (doneEmitted || closed) return;
+        doneEmitted = true;
+        emitter.emit("done");
+      };
+
+      const armQuiescence = () => {
+        clearQuiescence();
+        quiescenceTimer = setTimeout(emitDoneOnce, QUIESCENCE_MS);
+      };
+
+      const armFirstAudioTimer = () => {
+        clearQuiescence();
+        quiescenceTimer = setTimeout(emitDoneOnce, FIRST_AUDIO_TIMEOUT_MS);
+      };
+
+      ws.on("message", (raw: WebSocket.Data) => {
+        if (closed) return;
+        handleRimeMessage(raw, emitter, armQuiescence, () => quiescenceTimer !== null);
+      });
+
+      ws.on("error", (err: Error) => {
+        if (closed) return;
+        emitter.emit(
+          "error",
+          makeTtsError("tts_stream_error", `Rime TTS stream error: ${err?.message ?? String(err)}`),
+        );
+      });
+
+      ws.on("close", () => {
+        if (closed) return;
+        // Unexpected server-side close — surface `done` so the pipeline
+        // doesn't hang waiting for an utterance that will never complete.
+        emitDoneOnce();
+      });
+
+      const close = async (): Promise<void> => {
+        if (closed) return;
+        closed = true;
+        clearQuiescence();
+        try {
+          ws.close();
+        } catch {
+          // Swallow: caller has already decided to tear down.
+        }
+      };
+
+      if (openOpts.signal.aborted) {
+        void close();
+      } else {
+        openOpts.signal.addEventListener("abort", () => void close(), { once: true });
+      }
+
+      const session: RimeSession = {
+        sendText(text: string) {
+          if (closed || text.length === 0) return;
+          if (ws.readyState !== WebSocket.OPEN) return;
+          // Reset done state at the start of a new turn.
+          doneEmitted = false;
+          ws.send(JSON.stringify({ text }));
+        },
+
+        flush() {
+          if (closed) return;
+          if (ws.readyState !== WebSocket.OPEN) return;
+          // Force synthesis of any text buffered behind a missing terminal
+          // punctuation: append a trailing `"."`. Sending the JSON `eos`
+          // operation would close the connection, requiring a reconnect on
+          // every turn — `"."` keeps the WS reusable. Use the longer
+          // first-audio timer until the initial chunk arrives; the chunk
+          // handler swaps it for short quiescence on each subsequent chunk.
+          ws.send(JSON.stringify({ text: "." }));
+          armFirstAudioTimer();
+        },
+
+        cancel() {
+          if (closed) return;
+          // Drop Rime's server-side buffer for barge-in.
+          if (ws.readyState === WebSocket.OPEN) {
+            ws.send(JSON.stringify({ operation: "clear" }));
+          }
+          // Emit `done` synchronously — the orchestrator's state machine
+          // advances on `done`, and barge-in must not be microtask-deferred.
+          emitDoneOnce();
+        },
+
+        on(event, fn) {
+          return emitter.on(event, fn);
+        },
+
+        close,
+
+        _ws: ws,
+      };
+
+      return session;
+    },
+  };
+}

package/host/runtime-barrel.ts

@@ -17,8 +17,10 @@ export * from "./builtin-tools.ts";
 export * from "./runtime.ts";
 export * from "./runtime-config.ts";
 export * from "./server.ts";
-export * from "./session.ts";
-export * from "./session-ctx.ts";
+export * from "./session-core.ts";
 export * from "./tool-executor.ts";
+export * from "./transports/pipeline-transport.ts";
+export * from "./transports/s2s-transport.ts";
+export * from "./transports/types.ts";
 export * from "./unstorage-kv.ts";
 export * from "./ws-handler.ts";

package/host/runtime.test.ts

@@ -11,10 +11,10 @@ import {
   createFakeTtsProvider,
 } from "./_pipeline-test-fakes.ts";
 import { CONFORMANCE_AGENT, testRuntime } from "./_runtime-conformance.ts";
-import { flush, makeAgent, makeClient, makeMockHandle, silentLogger } from "./_test-utils.ts";
+import { flush, makeAgent, makeClientSink, makeMockHandle, silentLogger } from "./_test-utils.ts";
 import { createRuntime } from "./runtime.ts";
-import { _internals } from "./session.ts";
 import { executeToolCall } from "./tool-executor.ts";
+import { _internals } from "./transports/s2s-transport.ts";
 import { createUnstorageKv } from "./unstorage-kv.ts";
 
 describe("toAgentConfig", () => {
@@ -360,11 +360,7 @@ describe("createRuntime shutdown", () => {
 
   test("shutdown stops active sessions gracefully", async () => {
     const mockHandle = makeMockHandle();
-    const connectSpy = vi.spyOn(_internals, "connectS2s").mockImplementation(async () => {
-      // Fire "ready" so session.start() resolves
-      setTimeout(() => mockHandle._fire("ready", { sessionId: "mock-sid" }), 0);
-      return mockHandle;
-    });
+    const connectSpy = vi.spyOn(_internals, "connectS2s").mockResolvedValue(mockHandle);
 
     const agent = makeAgent();
     const runtime = createRuntime({ agent, env: {}, logger: silentLogger });
@@ -392,10 +388,7 @@ describe("createRuntime shutdown", () => {
     mockHandle.close = vi.fn(() => {
       throw new Error("close failed");
     });
-    const connectSpy = vi.spyOn(_internals, "connectS2s").mockImplementation(async () => {
-      setTimeout(() => mockHandle._fire("ready", { sessionId: "mock-sid" }), 0);
-      return mockHandle;
-    });
+    const connectSpy = vi.spyOn(_internals, "connectS2s").mockResolvedValue(mockHandle);
 
     const agent = makeAgent();
     const runtime = createRuntime({ agent, env: {}, logger });
@@ -422,10 +415,7 @@ describe("createRuntime shutdown", () => {
     mockHandle.close = vi.fn(() => {
       // intentionally do nothing — session stop will hang
     });
-    const connectSpy = vi.spyOn(_internals, "connectS2s").mockImplementation(async () => {
-      setTimeout(() => mockHandle._fire("ready", { sessionId: "mock-sid" }), 0);
-      return mockHandle;
-    });
+    const connectSpy = vi.spyOn(_internals, "connectS2s").mockResolvedValue(mockHandle);
 
     const agent = makeAgent();
     const runtime = createRuntime({
@@ -483,12 +473,7 @@ describe("createRuntime createSession", () => {
   test("createSession returns a Session object", () => {
     const agent = makeAgent();
     const runtime = createRuntime({ agent, env: {} });
-    const client = {
-      open: true,
-      event: vi.fn(),
-      playAudioChunk: vi.fn(),
-      playAudioDone: vi.fn(),
-    };
+    const client = makeClientSink();
     const session = runtime.createSession({
       id: "test-session",
       agent: agent.name,
@@ -506,12 +491,7 @@ describe("createRuntime createSession", () => {
   test("createSession passes skipGreeting option", () => {
     const agent = makeAgent();
     const runtime = createRuntime({ agent, env: {} });
-    const client = {
-      open: true,
-      event: vi.fn(),
-      playAudioChunk: vi.fn(),
-      playAudioDone: vi.fn(),
-    };
+    const client = makeClientSink();
     // Should not throw when skipGreeting is set
     const session = runtime.createSession({
       id: "test-session",
@@ -525,12 +505,7 @@ describe("createRuntime createSession", () => {
   test("createSession passes resumeFrom option", () => {
     const agent = makeAgent();
     const runtime = createRuntime({ agent, env: {} });
-    const client = {
-      open: true,
-      event: vi.fn(),
-      playAudioChunk: vi.fn(),
-      playAudioDone: vi.fn(),
-    };
+    const client = makeClientSink();
     const session = runtime.createSession({
       id: "test-session",
       agent: agent.name,
@@ -604,13 +579,8 @@ describe("createRuntime with custom options", () => {
       agent,
       env: { ASSEMBLYAI_API_KEY: "test-api-key" },
     });
-    const client = {
-      open: true,
-      event: vi.fn(),
-      playAudioChunk: vi.fn(),
-      playAudioDone: vi.fn(),
-    };
-    // Should not throw — the API key gets passed to createS2sSession internally
+    const client = makeClientSink();
+    // Should not throw — the API key gets passed to createS2sTransport internally
     const session = runtime.createSession({
       id: "test-session",
       agent: agent.name,
@@ -651,7 +621,7 @@ describe("Runtime — session routing", () => {
       tts,
     });
 
-    const client = makeClient();
+    const client = makeClientSink();
     const session = runtime.createSession({
       id: "sess-pipeline",
       agent: "test-agent",
@@ -677,10 +647,7 @@ describe("Runtime — session routing", () => {
 
   test("manifest without stt/llm/tts routes to S2sSession (createWebSocket IS called)", async () => {
     const mockHandle = makeMockHandle();
-    const connectSpy = vi.spyOn(_internals, "connectS2s").mockImplementation(async () => {
-      setTimeout(() => mockHandle._fire("ready", { sessionId: "mock-sid" }), 0);
-      return mockHandle;
-    });
+    const connectSpy = vi.spyOn(_internals, "connectS2s").mockResolvedValue(mockHandle);
 
     const createWebSocket = vi.fn();
     const runtime = createRuntime({
@@ -690,7 +657,7 @@ describe("Runtime — session routing", () => {
       createWebSocket,
     });
 
-    const client = makeClient();
+    const client = makeClientSink();
     const session = runtime.createSession({
       id: "sess-s2s",
       agent: "test-agent",

package/host/runtime.ts

@@ -15,6 +15,8 @@ import { DEFAULT_SHUTDOWN_TIMEOUT_MS } from "../sdk/constants.ts";
 import type { Kv } from "../sdk/kv.ts";
 import type { ClientSink } from "../sdk/protocol.ts";
 import { buildReadyConfig, type ReadyConfig } from "../sdk/protocol.ts";
+import { DEEPGRAM_KIND } from "../sdk/providers/stt/deepgram.ts";
+import { RIME_KIND } from "../sdk/providers/tts/rime.ts";
 import {
   assertProviderTriple,
   type LlmProvider,
@@ -23,19 +25,68 @@ import {
   type TtsOpener,
   type TtsProvider,
 } from "../sdk/providers.ts";
+import { buildSystemPrompt } from "../sdk/system-prompt.ts";
 import type { AgentDef } from "../sdk/types.ts";
 import { toolError } from "../sdk/utils.ts";
 import { resolveAllBuiltins } from "./builtin-tools.ts";
-import { createPipelineSession } from "./pipeline-session.ts";
 import { resolveApiKey, resolveLlm, resolveStt, resolveTts } from "./providers/resolve.ts";
 import type { Logger, S2SConfig } from "./runtime-config.ts";
 import { consoleLogger, DEFAULT_S2S_CONFIG } from "./runtime-config.ts";
 import type { CreateS2sWebSocket } from "./s2s.ts";
-import { createS2sSession, type Session } from "./session.ts";
+import { createSessionCore, type SessionCore } from "./session-core.ts";
 import { type ExecuteTool, executeToolCall } from "./tool-executor.ts";
+import { createPipelineTransport } from "./transports/pipeline-transport.ts";
+import { createS2sTransport } from "./transports/s2s-transport.ts";
+import type { Transport, TransportCallbacks } from "./transports/types.ts";
 import { createUnstorageKv } from "./unstorage-kv.ts";
 import { type SessionWebSocket, wireSessionSocket } from "./ws-handler.ts";
 
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+
+/**
+ * Resolve the API key env-var for the configured STT provider.
+ *
+ * Each STT provider uses its own env var (e.g. `ASSEMBLYAI_API_KEY`,
+ * `DEEPGRAM_API_KEY`). We read the kind from the descriptor if it is one;
+ * pre-resolved openers have no kind field so we fall back to AssemblyAI for
+ * backward compatibility (openers supply their own key at open-time anyway).
+ */
+function resolveSttApiKey(
+  stt: SttProvider | SttOpener | undefined,
+  env: Record<string, string>,
+): string {
+  // SttProvider descriptors carry a `kind` field; SttOpener does not.
+  const kind =
+    stt != null && "kind" in stt && typeof (stt as SttProvider).kind === "string"
+      ? (stt as SttProvider).kind
+      : undefined;
+  if (kind === DEEPGRAM_KIND) return resolveApiKey("DEEPGRAM_API_KEY", env);
+  // Default: ASSEMBLYAI_KIND or pre-resolved opener (backward compat).
+  return resolveApiKey("ASSEMBLYAI_API_KEY", env);
+}
+
+/**
+ * Resolve the API key env-var for the configured TTS provider.
+ *
+ * Each TTS provider uses its own env var (e.g. `CARTESIA_API_KEY`,
+ * `RIME_API_KEY`). We read the kind from the descriptor if it is one;
+ * pre-resolved openers have no kind field so we fall back to Cartesia for
+ * backward compatibility (openers supply their own key at open-time anyway).
+ */
+function resolveTtsApiKey(
+  tts: TtsProvider | TtsOpener | undefined,
+  env: Record<string, string>,
+): string {
+  // TtsProvider descriptors carry a `kind` field; TtsOpener does not.
+  const kind =
+    tts != null && "kind" in tts && typeof (tts as TtsProvider).kind === "string"
+      ? (tts as TtsProvider).kind
+      : undefined;
+  if (kind === RIME_KIND) return resolveApiKey("RIME_API_KEY", env);
+  // Default: CARTESIA_KIND or pre-resolved opener (backward compat).
+  return resolveApiKey("CARTESIA_API_KEY", env);
+}
+
 // ─── Runtime adapter (formerly adapter.ts) ──────────────────────────────────
 
 /** Per-session options passed to {@link AgentRuntime.startSession}. */
@@ -189,7 +240,7 @@ export type Runtime = AgentRuntime & {
     client: ClientSink;
     skipGreeting?: boolean;
     resumeFrom?: string;
-  }): Session;
+  }): SessionCore;
 };
 
 /**
@@ -217,7 +268,7 @@ export function createRuntime(opts: RuntimeOptions): Runtime {
   } = opts;
   const mode = assertProviderTriple(opts.stt, opts.llm, opts.tts);
   const agentConfig = toAgentConfig(agent);
-  const sessions = new Map<string, Session>();
+  const sessions = new Map<string, SessionCore>();
   const sinkMap = new Map<string, ClientSink>();
   const readyConfig: ReadyConfig = buildReadyConfig(s2sConfig);
 
@@ -310,44 +361,101 @@ export function createRuntime(opts: RuntimeOptions): Runtime {
     client: ClientSink;
     skipGreeting?: boolean;
     resumeFrom?: string;
-  }): Session {
+  }): SessionCore {
     sinkMap.set(sessionOpts.id, sessionOpts.client);
+
+    const isPipeline = Boolean(pipelineProviders);
+    const hasTools = toolSchemas.length > 0 || (agentConfig.builtinTools?.length ?? 0) > 0;
+    const systemPrompt = buildSystemPrompt(agentConfig, {
+      hasTools,
+      voice: true,
+      toolGuidance,
+    });
+
+    // Late-bound reference: callbacks are constructed before SessionCore exists,
+    // so we capture a reference and fill it in below.
+    let core: SessionCore | null = null;
+    function bindCore(): SessionCore {
+      if (!core) throw new Error("SessionCore not yet created");
+      return core;
+    }
+
+    const callbacks: TransportCallbacks = {
+      onReplyStarted: (replyId) => bindCore().onReplyStarted(replyId),
+      onReplyDone: () => bindCore().onReplyDone(),
+      onCancelled: () => bindCore().onCancelled(),
+      onAudioChunk: (bytes) => bindCore().onAudioChunk(bytes),
+      onAudioDone: () => bindCore().onAudioDone(),
+      onUserTranscript: (text) => bindCore().onUserTranscript(text),
+      onAgentTranscript: (text, interrupted) => bindCore().onAgentTranscript(text, interrupted),
+      // Pipeline: tools execute inside streamText; forward the call to the
+      // client sink for UI observability only. Going through SessionCore.onToolCall
+      // would re-execute the tool and leave pendingTools non-empty, hanging the turn.
+      onToolCall: isPipeline
+        ? (id, name, args) =>
+            sessionOpts.client.event({ type: "tool_call", toolCallId: id, toolName: name, args })
+        : (id, name, args) => bindCore().onToolCall(id, name, args),
+      onError: (code, message) => bindCore().onError(code, message),
+      onSpeechStarted: () => bindCore().onSpeechStarted(),
+      onSpeechStopped: () => bindCore().onSpeechStopped(),
+    };
+
+    let transport: Transport;
     if (pipelineProviders) {
-      return createPipelineSession({
-        id: sessionOpts.id,
+      transport = createPipelineTransport({
+        sid: sessionOpts.id,
         agent: sessionOpts.agent,
-        client: sessionOpts.client,
-        agentConfig,
-        toolSchemas,
-        toolGuidance,
-        executeTool,
         stt: pipelineProviders.stt,
         llm: pipelineProviders.llm,
         tts: pipelineProviders.tts,
-        sttApiKey: resolveApiKey("ASSEMBLYAI_API_KEY", env),
-        ttsApiKey: resolveApiKey("CARTESIA_API_KEY", env),
+        callbacks,
+        sessionConfig: {
+          systemPrompt,
+          greeting: agentConfig.greeting,
+          tools: toolSchemas,
+        },
+        toolSchemas,
+        executeTool,
+        providerKeys: {
+          stt: resolveSttApiKey(opts.stt, env),
+          tts: resolveTtsApiKey(opts.tts, env),
+        },
         sttSampleRate: s2sConfig.inputSampleRate,
         ttsSampleRate: s2sConfig.outputSampleRate,
+        maxSteps: agentConfig.maxSteps,
+        toolChoice: agentConfig.toolChoice,
         skipGreeting: sessionOpts.skipGreeting ?? false,
         logger,
       });
+    } else {
+      transport = createS2sTransport({
+        apiKey: env.ASSEMBLYAI_API_KEY ?? "",
+        s2sConfig,
+        sessionConfig: {
+          systemPrompt,
+          tools: toolSchemas as import("./s2s.ts").S2sToolSchema[],
+          ...(agentConfig.greeting !== undefined ? { greeting: agentConfig.greeting } : {}),
+        },
+        toolSchemas: toolSchemas as import("./s2s.ts").S2sToolSchema[],
+        callbacks,
+        sid: sessionOpts.id,
+        agent: sessionOpts.agent,
+        ...(createWebSocket ? { createWebSocket } : {}),
+        logger,
+      });
     }
-    const apiKey = env.ASSEMBLYAI_API_KEY ?? "";
-    return createS2sSession({
+
+    core = createSessionCore({
       id: sessionOpts.id,
       agent: sessionOpts.agent,
       client: sessionOpts.client,
       agentConfig,
-      toolSchemas,
-      toolGuidance,
-      apiKey,
-      s2sConfig,
       executeTool,
-      ...(createWebSocket ? { createWebSocket } : {}),
-      skipGreeting: sessionOpts.skipGreeting ?? false,
+      transport,
       logger,
-      ...(sessionOpts.resumeFrom ? { resumeFrom: sessionOpts.resumeFrom } : {}),
     });
+
+    return core;
   }
 
   // ── AgentRuntime methods ──────────────────────────────────────────────