@alexkroman1/aai 1.5.1 → 1.6.1

package/host/providers/stt/soniox.ts

@@ -0,0 +1,239 @@
+// Copyright 2026 the AAI authors. MIT license.
+/**
+ * Soniox real-time STT opener (host-only).
+ *
+ * The user-facing descriptor factory (`soniox(...)`) lives in
+ * `sdk/providers/stt/soniox.ts`. This module is the host-side
+ * counterpart: it takes the descriptor options + an API key and
+ * returns an {@link SttOpener} that the pipeline session drives.
+ *
+ * Soniox's published JS client (`@soniox/speech-to-text-web`) is
+ * browser-only — it depends on `MediaRecorder` and `getUserMedia`. For
+ * server-side use we talk to the WebSocket directly:
+ *   `wss://stt-rt.soniox.com/transcribe-websocket`
+ *
+ * Wire format:
+ *   - First text frame: JSON config with api_key, model, audio_format,
+ *     sample_rate, num_channels (and optional language hints).
+ *   - Subsequent binary frames: 16-bit signed little-endian PCM audio.
+ *   - Server replies: JSON `{ tokens: [{ text, is_final }] }` messages.
+ *     Final tokens accumulate; non-final tokens are a rolling preview.
+ *   - On error: `{ error_code, error_message }`.
+ */
+
+import { createNanoEvents, type Emitter } from "nanoevents";
+import WebSocket from "ws";
+import type { SonioxOptions } from "../../../sdk/providers/stt/soniox.ts";
+import {
+  makeSttError,
+  type SttEvents,
+  type SttOpener,
+  type SttOpenOptions,
+  type SttSession,
+} from "../../../sdk/providers.ts";
+
+const SONIOX_WS_URL = "wss://stt-rt.soniox.com/transcribe-websocket";
+
+/** Soniox token shape from the wire protocol. */
+interface SonioxToken {
+  text?: string;
+  is_final?: boolean;
+}
+
+interface SonioxResponse {
+  tokens?: SonioxToken[];
+  finished?: boolean;
+  error_code?: number;
+  error_message?: string;
+}
+
+/**
+ * Walk a batch of Soniox tokens, sending finals into `appendFinal` and
+ * returning the concatenated non-finals as a rolling preview string.
+ */
+function consumeTokens(tokens: SonioxToken[], appendFinal: (text: string) => void): string {
+  let nonFinal = "";
+  for (const tok of tokens) {
+    const text = tok.text ?? "";
+    if (text.length === 0) continue;
+    if (tok.is_final) {
+      appendFinal(text);
+    } else {
+      nonFinal += text;
+    }
+  }
+  return nonFinal;
+}
+
+/** Resolve once the WebSocket opens; reject on the first error. */
+function waitForOpen(ws: WebSocket): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const onOpen = () => {
+      ws.off("error", onErr);
+      resolve();
+    };
+    const onErr = (err: Error) => {
+      ws.off("open", onOpen);
+      reject(err);
+    };
+    ws.once("open", onOpen);
+    ws.once("error", onErr);
+  });
+}
+
+/** Build the initial JSON config frame for a Soniox session. */
+function buildConfigFrame(
+  apiKey: string,
+  opts: SonioxOptions,
+  sampleRate: number,
+): Record<string, unknown> {
+  const config: Record<string, unknown> = {
+    api_key: apiKey,
+    model: opts.model ?? "stt-rt-v3",
+    audio_format: "pcm_s16le",
+    sample_rate: sampleRate,
+    num_channels: 1,
+  };
+  if (opts.languageHints && opts.languageHints.length > 0) {
+    config.language_hints = [...opts.languageHints];
+  }
+  return config;
+}
+
+/** Parse a Soniox text frame into a {@link SonioxResponse}; returns null on garbage. */
+function parseFrame(raw: WebSocket.RawData): SonioxResponse | null {
+  try {
+    return JSON.parse(raw.toString()) as SonioxResponse;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Handle one server response. Emits `error`, `final`, and `partial` events
+ * onto `emitter` based on the token batch and the running `finalBuf`. The
+ * caller owns `finalBuf` so it survives across messages and can be flushed
+ * on close.
+ */
+function handleResponse(
+  res: SonioxResponse,
+  emitter: Emitter<SttEvents>,
+  finalBuf: { value: string },
+): void {
+  if (res.error_code !== undefined) {
+    emitter.emit(
+      "error",
+      makeSttError(
+        "stt_stream_error",
+        `Soniox error ${res.error_code}: ${res.error_message ?? "unknown"}`,
+      ),
+    );
+    return;
+  }
+  if (!res.tokens || res.tokens.length === 0) return;
+  const nonFinal = consumeTokens(res.tokens, (text) => {
+    finalBuf.value += text;
+  });
+  // Flush an accumulated final whenever the next batch's non-final preview
+  // begins (or when the session finishes). This batches contiguous final
+  // tokens into a single `final` event, matching what downstream pipeline
+  // session code expects.
+  if (finalBuf.value.length > 0 && (nonFinal.length > 0 || res.finished)) {
+    emitter.emit("final", finalBuf.value);
+    finalBuf.value = "";
+  }
+  if (nonFinal.length > 0) {
+    emitter.emit("partial", nonFinal);
+  }
+}
+
+/** Build an {@link SttOpener} from resolved Soniox descriptor options. */
+export function openSoniox(opts: SonioxOptions = {}): SttOpener {
+  return {
+    name: "soniox",
+    async open(openOpts: SttOpenOptions): Promise<SttSession> {
+      const apiKey = openOpts.apiKey || process.env.SONIOX_API_KEY;
+      if (!apiKey) {
+        throw makeSttError(
+          "stt_auth_failed",
+          "Soniox STT: missing API key. Set SONIOX_API_KEY in the agent env.",
+        );
+      }
+
+      const ws = new WebSocket(SONIOX_WS_URL);
+      const emitter: Emitter<SttEvents> = createNanoEvents<SttEvents>();
+      let closed = false;
+      // Soniox emits final tokens once and non-final tokens repeatedly. We
+      // accumulate finals into a buffer flushed on each non-final boundary
+      // and forward non-finals as the rolling partial. Mirrors how the
+      // existing AssemblyAI/Deepgram openers map provider-specific token
+      // streams onto the SttEvents `partial`/`final` contract.
+      const finalBuf = { value: "" };
+
+      try {
+        await waitForOpen(ws);
+      } catch (cause) {
+        throw makeSttError(
+          "stt_connect_failed",
+          `Soniox STT: connect failed: ${cause instanceof Error ? cause.message : String(cause)}`,
+        );
+      }
+
+      // Initial config frame (text). Sent first; audio binary frames follow.
+      ws.send(JSON.stringify(buildConfigFrame(apiKey, opts, openOpts.sampleRate)));
+
+      ws.on("message", (raw: WebSocket.RawData) => {
+        if (closed) return;
+        const res = parseFrame(raw);
+        if (res) handleResponse(res, emitter, finalBuf);
+      });
+
+      ws.on("error", (err: Error) => {
+        if (closed) return;
+        emitter.emit("error", makeSttError("stt_stream_error", err.message ?? String(err)));
+      });
+
+      ws.on("close", (code: number) => {
+        if (closed) return;
+        // 1000 = normal closure.
+        if (code !== 1000) {
+          emitter.emit("error", makeSttError("stt_stream_error", `socket closed ${code}`));
+        }
+      });
+
+      const close = async (): Promise<void> => {
+        if (closed) return;
+        closed = true;
+        // Flush any trailing final tokens that arrived right before close.
+        if (finalBuf.value.length > 0) {
+          emitter.emit("final", finalBuf.value);
+          finalBuf.value = "";
+        }
+        try {
+          ws.close();
+        } catch {
+          // Swallow: caller has already decided to tear down.
+        }
+      };
+
+      if (openOpts.signal.aborted) {
+        void close();
+      } else {
+        openOpts.signal.addEventListener("abort", () => void close(), { once: true });
+      }
+
+      return {
+        sendAudio(pcm: Int16Array) {
+          if (closed || ws.readyState !== WebSocket.OPEN) return;
+          // Sending the underlying buffer directly avoids a copy. ws will
+          // hand it to the OS as a binary frame.
+          ws.send(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength), { binary: true });
+        },
+        on(event, fn) {
+          return emitter.on(event, fn);
+        },
+        close,
+      };
+    },
+  };
+}

package/host/transports/pipeline-transport.test.ts

@@ -210,6 +210,97 @@ describe("PipelineTransport", () => {
       await t.stop();
     });
 
+    test("inserts a separator between text segments split by a mid-turn tool call", async () => {
+      // Multi-step turn: step 1 ends with a text segment + tool-call, step 2
+      // begins with a fresh text segment. Without the fix, the deltas fuse
+      // into "...up.Got it" — both in the transcript and in TTS input.
+      const stt = createFakeSttProvider();
+      const tts = createFakeTtsProvider();
+      const callbacks = makeCallbacks();
+      const executeTool = vi.fn(async () => "result");
+      const { opts } = makeOpts(
+        {
+          llm: createFakeLanguageModel({
+            steps: [
+              [
+                { type: "text", text: "Let me look that up." },
+                { type: "tool-call", toolCallId: "tc-1", toolName: "lookup", input: "{}" },
+              ],
+              [{ type: "text", text: "Got it. Here's the answer." }],
+            ],
+          }),
+          executeTool,
+          toolSchemas: [
+            {
+              type: "function" as const,
+              name: "lookup",
+              description: "Look something up.",
+              parameters: { type: "object" as const, properties: {}, required: [] },
+            },
+          ],
+          sessionConfig: { systemPrompt: "s", greeting: "" },
+        },
+        { stt, tts, callbacks },
+      );
+      const t = createPipelineTransport(opts);
+      await t.start();
+      stt.last()?.fireFinal("look it up");
+      await vi.waitFor(() => {
+        expect(callbacks.onAgentTranscript).toHaveBeenCalled();
+      });
+      expect(callbacks.onAgentTranscript).toHaveBeenCalledWith(
+        "Let me look that up. Got it. Here's the answer.",
+        false,
+      );
+      expect(tts.last()?.textChunks.join("")).toBe(
+        "Let me look that up. Got it. Here's the answer.",
+      );
+      await t.stop();
+    });
+
+    test("does not double-space when a segment boundary already carries whitespace", async () => {
+      // Trailing space on segment 1 — we must not insert an extra space.
+      const stt = createFakeSttProvider();
+      const tts = createFakeTtsProvider();
+      const callbacks = makeCallbacks();
+      const executeTool = vi.fn(async () => "result");
+      const { opts } = makeOpts(
+        {
+          llm: createFakeLanguageModel({
+            steps: [
+              [
+                { type: "text", text: "First sentence. " },
+                { type: "tool-call", toolCallId: "tc-1", toolName: "lookup", input: "{}" },
+              ],
+              [{ type: "text", text: "Second sentence." }],
+            ],
+          }),
+          executeTool,
+          toolSchemas: [
+            {
+              type: "function" as const,
+              name: "lookup",
+              description: "Look something up.",
+              parameters: { type: "object" as const, properties: {}, required: [] },
+            },
+          ],
+          sessionConfig: { systemPrompt: "s", greeting: "" },
+        },
+        { stt, tts, callbacks },
+      );
+      const t = createPipelineTransport(opts);
+      await t.start();
+      stt.last()?.fireFinal("look it up");
+      await vi.waitFor(() => {
+        expect(callbacks.onAgentTranscript).toHaveBeenCalled();
+      });
+      expect(callbacks.onAgentTranscript).toHaveBeenCalledWith(
+        "First sentence. Second sentence.",
+        false,
+      );
+      await t.stop();
+    });
+
     test("TTS audio event is forwarded to callbacks.onAudioChunk as Uint8Array", async () => {
       const stt = createFakeSttProvider();
       const tts = createFakeTtsProvider();

package/host/transports/pipeline-transport.ts

@@ -217,9 +217,10 @@ export function createPipelineTransport(opts: PipelineTransportOptions): Transpo
         stopWhen: stepCountIs(maxSteps),
         abortSignal: ctl.signal,
       });
+      const handlePart = makeStreamPartHandler(onDelta);
       for await (const part of result.fullStream) {
         if (ctl.signal.aborted) break;
-        handleStreamPart(part, ctl, onDelta);
+        handlePart(part);
       }
     } catch (err: unknown) {
       if (!ctl.signal.aborted) {
@@ -230,8 +231,33 @@ export function createPipelineTransport(opts: PipelineTransportOptions): Transpo
     }
   }
 
-  function handleStreamPart(
-    part: {
+  /**
+   * Stateful per-turn handler for `streamText` `fullStream` parts.
+   *
+   * Tracks text-segment boundaries so that consecutive segments — which the
+   * Vercel SDK emits across tool-call hops as `text-end` followed later by a
+   * fresh `text-start` — don't fuse into "...up.Got it" when concatenated for
+   * the transcript or streamed to TTS. When a boundary is crossed and neither
+   * side carries whitespace, a single space is injected into both streams.
+   */
+  function makeStreamPartHandler(onDelta: (delta: string) => void) {
+    let pendingSeparator = false;
+    let lastChar = "";
+
+    function emitText(delta: string): void {
+      if (delta.length === 0) return;
+      let out = delta;
+      if (pendingSeparator) {
+        pendingSeparator = false;
+        const boundaryHasSpace = lastChar === "" || /\s/.test(lastChar) || /^\s/.test(out);
+        if (!boundaryHasSpace) out = ` ${out}`;
+      }
+      lastChar = out.slice(-1);
+      onDelta(out);
+      ttsSession?.sendText(out);
+    }
+
+    return function handlePart(part: {
       readonly type: string;
       readonly text?: string;
       readonly input?: unknown;
@@ -239,34 +265,31 @@ export function createPipelineTransport(opts: PipelineTransportOptions): Transpo
       readonly toolCallId?: string;
       readonly toolName?: string;
       readonly error?: unknown;
-    },
-    _ctl: AbortController,
-    onDelta: (delta: string) => void,
-  ): void {
-    switch (part.type) {
-      case "text-delta": {
-        const delta = part.text ?? "";
-        if (delta.length === 0) return;
-        onDelta(delta);
-        ttsSession?.sendText(delta);
-        return;
-      }
-      case "tool-call": {
-        // Option A: fire callbacks.onToolCall for observability only.
-        // Actual execution happens inline via toVercelTools.
-        const input = (part.input ?? {}) as Record<string, unknown>;
-        callbacks.onToolCall(part.toolCallId ?? "", part.toolName ?? "", input);
-        return;
-      }
-      case "error": {
-        const msg = errorMessage(part.error);
-        log.error("LLM stream error", { message: msg, sid: opts.sid });
-        emitError("llm", msg);
-        return;
+    }): void {
+      switch (part.type) {
+        case "text-delta":
+          emitText(part.text ?? "");
+          return;
+        case "text-end":
+          pendingSeparator = true;
+          return;
+        case "tool-call": {
+          // Option A: fire callbacks.onToolCall for observability only.
+          // Actual execution happens inline via toVercelTools.
+          const input = (part.input ?? {}) as Record<string, unknown>;
+          callbacks.onToolCall(part.toolCallId ?? "", part.toolName ?? "", input);
+          return;
+        }
+        case "error": {
+          const msg = errorMessage(part.error);
+          log.error("LLM stream error", { message: msg, sid: opts.sid });
+          emitError("llm", msg);
+          return;
+        }
+        default:
+          return;
       }
-      default:
-        return;
-    }
+    };
   }
 
   // ---- TTS flush ------------------------------------------------------------

package/host/transports/s2s-transport.test.ts

@@ -1,5 +1,7 @@
-import { describe, expect, test, vi } from "vitest";
-import { createS2sTransport } from "./s2s-transport.ts";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import { makeMockHandle, silentLogger } from "../_test-utils.ts";
+import type { S2sCallbacks, S2sHandle } from "../s2s.ts";
+import { _internals, createS2sTransport } from "./s2s-transport.ts";
 import type { TransportCallbacks } from "./types.ts";
 
 function makeCallbacks(): TransportCallbacks {
@@ -15,6 +17,7 @@ function makeCallbacks(): TransportCallbacks {
     onError: vi.fn(),
     onSpeechStarted: vi.fn(),
     onSpeechStopped: vi.fn(),
+    onSessionReady: vi.fn(),
   };
 }
 
@@ -54,3 +57,220 @@ describe("S2sTransport", () => {
     expect(close).toHaveBeenCalled();
   });
 });
+
+// ─── Reconnect tests ────────────────────────────────────────────────────────
+
+/** Capture the S2sCallbacks that the transport hands to connectS2s. */
+function setupSpiedTransport(): {
+  callbacks: TransportCallbacks;
+  handles: S2sHandle[];
+  capturedCallbacks: S2sCallbacks[];
+  spy: ReturnType<typeof vi.spyOn>;
+} {
+  const handles: S2sHandle[] = [];
+  const capturedCallbacks: S2sCallbacks[] = [];
+  const spy = vi
+    .spyOn(_internals, "connectS2s")
+    .mockImplementation(async (opts: import("../s2s.ts").ConnectS2sOptions) => {
+      capturedCallbacks.push(opts.callbacks);
+      const h = makeMockHandle();
+      handles.push(h);
+      return h;
+    });
+  return {
+    callbacks: makeCallbacks(),
+    handles,
+    capturedCallbacks,
+    spy,
+  };
+}
+
+describe("S2sTransport reconnect", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  test("attempts session.resume on transient close (1005) inside the resume window", async () => {
+    const { callbacks, handles, capturedCallbacks } = setupSpiedTransport();
+
+    const t = createS2sTransport({
+      apiKey: "k",
+      s2sConfig: { wssUrl: "wss://fake", inputSampleRate: 16_000, outputSampleRate: 24_000 },
+      sessionConfig: { systemPrompt: "test", tools: [] },
+      toolSchemas: [],
+      callbacks,
+      sid: "sid-1",
+      agent: "a",
+      logger: silentLogger,
+    });
+    await t.start();
+
+    // Establish session, start a reply, then drop the socket.
+    const cb1 = capturedCallbacks[0];
+    if (!cb1) throw new Error("expected first callbacks");
+    cb1.onSessionReady("sess_abc");
+    cb1.onReplyStarted("rep_1");
+    cb1.onClose(1005, "");
+
+    // Wait for the async resume() to fire connectS2s a second time.
+    await vi.waitFor(() => {
+      expect(handles.length).toBe(2);
+    });
+
+    // The new handle should have received resumeSession with the prior id.
+    const newHandle = handles[1];
+    if (!newHandle) throw new Error("expected new handle");
+    expect(newHandle.resumeSession).toHaveBeenCalledWith("sess_abc");
+
+    // The in-flight reply was unblocked via onCancelled, NOT a fatal error.
+    expect(callbacks.onCancelled).toHaveBeenCalledOnce();
+    expect(callbacks.onError).not.toHaveBeenCalled();
+  });
+
+  test("does NOT reconnect on fatal close codes (1008 unauthorized)", async () => {
+    const { callbacks, handles, capturedCallbacks } = setupSpiedTransport();
+
+    const t = createS2sTransport({
+      apiKey: "k",
+      s2sConfig: { wssUrl: "wss://fake", inputSampleRate: 16_000, outputSampleRate: 24_000 },
+      sessionConfig: { systemPrompt: "test", tools: [] },
+      toolSchemas: [],
+      callbacks,
+      sid: "sid-1",
+      agent: "a",
+      logger: silentLogger,
+    });
+    await t.start();
+
+    const cb1 = capturedCallbacks[0];
+    if (!cb1) throw new Error("expected first callbacks");
+    cb1.onSessionReady("sess_abc");
+    cb1.onReplyStarted("rep_1");
+    cb1.onClose(1008, "unauthorized");
+
+    // No reconnect — only one connectS2s call total.
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    expect(handles.length).toBe(1);
+    // Fatal error surfaces, since a reply was in flight.
+    expect(callbacks.onError).toHaveBeenCalledWith(
+      "connection",
+      expect.stringContaining("S2S closed mid-reply"),
+    );
+  });
+
+  test("does NOT reconnect when stop() was called", async () => {
+    const { callbacks, handles, capturedCallbacks } = setupSpiedTransport();
+
+    const t = createS2sTransport({
+      apiKey: "k",
+      s2sConfig: { wssUrl: "wss://fake", inputSampleRate: 16_000, outputSampleRate: 24_000 },
+      sessionConfig: { systemPrompt: "test", tools: [] },
+      toolSchemas: [],
+      callbacks,
+      sid: "sid-1",
+      agent: "a",
+      logger: silentLogger,
+    });
+    await t.start();
+
+    const cb1 = capturedCallbacks[0];
+    if (!cb1) throw new Error("expected first callbacks");
+    cb1.onSessionReady("sess_abc");
+    await t.stop();
+
+    // Simulate the upstream's close arriving after stop() — it should be
+    // treated as a clean shutdown, not a transient drop worth resuming.
+    cb1.onClose(1005, "");
+
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    expect(handles.length).toBe(1);
+    expect(callbacks.onError).not.toHaveBeenCalled();
+  });
+
+  test("surfaces resume failure when the resumed socket also closes", async () => {
+    const { callbacks, handles, capturedCallbacks } = setupSpiedTransport();
+
+    const t = createS2sTransport({
+      apiKey: "k",
+      s2sConfig: { wssUrl: "wss://fake", inputSampleRate: 16_000, outputSampleRate: 24_000 },
+      sessionConfig: { systemPrompt: "test", tools: [] },
+      toolSchemas: [],
+      callbacks,
+      sid: "sid-1",
+      agent: "a",
+      logger: silentLogger,
+    });
+    await t.start();
+
+    capturedCallbacks[0]?.onSessionReady("sess_abc");
+    capturedCallbacks[0]?.onReplyStarted("rep_1");
+    capturedCallbacks[0]?.onClose(1005, "");
+
+    await vi.waitFor(() => expect(handles.length).toBe(2));
+
+    // The resume socket also drops before its session.ready arrives.
+    const cb2 = capturedCallbacks[1];
+    if (!cb2) throw new Error("expected resume callbacks");
+    cb2.onClose(1006, "");
+
+    expect(callbacks.onError).toHaveBeenCalledWith(
+      "connection",
+      expect.stringContaining("resume failed"),
+    );
+  });
+
+  test("surfaces resume failure when server reports session_not_found", async () => {
+    const { callbacks, handles, capturedCallbacks } = setupSpiedTransport();
+
+    const t = createS2sTransport({
+      apiKey: "k",
+      s2sConfig: { wssUrl: "wss://fake", inputSampleRate: 16_000, outputSampleRate: 24_000 },
+      sessionConfig: { systemPrompt: "test", tools: [] },
+      toolSchemas: [],
+      callbacks,
+      sid: "sid-1",
+      agent: "a",
+      logger: silentLogger,
+    });
+    await t.start();
+
+    capturedCallbacks[0]?.onSessionReady("sess_abc");
+    capturedCallbacks[0]?.onClose(1005, "");
+
+    await vi.waitFor(() => expect(handles.length).toBe(2));
+
+    capturedCallbacks[1]?.onSessionExpired();
+
+    expect(callbacks.onError).toHaveBeenCalledWith(
+      "connection",
+      expect.stringContaining("session expired"),
+    );
+  });
+
+  test("after a successful resume, a later transient drop also resumes", async () => {
+    const { callbacks, handles, capturedCallbacks } = setupSpiedTransport();
+
+    const t = createS2sTransport({
+      apiKey: "k",
+      s2sConfig: { wssUrl: "wss://fake", inputSampleRate: 16_000, outputSampleRate: 24_000 },
+      sessionConfig: { systemPrompt: "test", tools: [] },
+      toolSchemas: [],
+      callbacks,
+      sid: "sid-1",
+      agent: "a",
+      logger: silentLogger,
+    });
+    await t.start();
+
+    // First connection establishes, drops, resumes, becomes ready again.
+    capturedCallbacks[0]?.onSessionReady("sess_abc");
+    capturedCallbacks[0]?.onClose(1005, "");
+    await vi.waitFor(() => expect(handles.length).toBe(2));
+    capturedCallbacks[1]?.onSessionReady("sess_abc");
+
+    // Second drop — should trigger another resume attempt.
+    capturedCallbacks[1]?.onClose(1006, "");
+    await vi.waitFor(() => expect(handles.length).toBe(3));
+    expect(handles[2]?.resumeSession).toHaveBeenCalledWith("sess_abc");
+  });
+});