@kuralle-syrinx/server-websocket 2.1.0

package/src/browser-pacing.test.ts

@@ -0,0 +1,440 @@
+// SPDX-License-Identifier: MIT
+// WT-03: Browser outbound pacing + playout clock + client jitter buffer
+
+import { createServer } from "node:http";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import WebSocket from "ws";
+import { Route, VoiceAgentSession } from "@kuralle-syrinx/core";
+import { pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
+import { createVoiceWebSocketServer, type VoiceWebSocketServer } from "./index.js";
+
+function browserUrl(port: number): string {
+  return `ws://127.0.0.1:${port}/ws`;
+}
+
+async function openBrowserSocketReady(url: string): Promise<WebSocket> {
+  const socket = new WebSocket(url);
+  await new Promise<void>((resolve, reject) => {
+    const onMessage = (data: WebSocket.RawData, isBinary: boolean): void => {
+      if (isBinary) return;
+      try {
+        const parsed = JSON.parse(data.toString()) as { type?: string };
+        if (parsed.type === "ready") {
+          socket.off("message", onMessage);
+          resolve();
+        }
+      } catch {
+        // Ignore parsing errors
+      }
+    };
+    socket.on("message", onMessage);
+    socket.once("error", reject);
+  });
+  return socket;
+}
+
+function waitMs(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForCondition(predicate: () => boolean, timeoutMs = 1000): Promise<void> {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (predicate()) return;
+    await waitMs(10);
+  }
+  throw new Error("Timed out waiting for browser pacing condition");
+}
+
+async function readJsonMatching(socket: WebSocket, predicate: (m: unknown) => boolean): Promise<unknown> {
+  return new Promise((resolve) => {
+    const onMessage = (data: WebSocket.RawData, isBinary: boolean) => {
+      if (isBinary) return;
+      const message = JSON.parse(data.toString()) as unknown;
+      if (!predicate(message)) return;
+      socket.off("message", onMessage);
+      resolve(message);
+    };
+    socket.on("message", onMessage);
+  });
+}
+
+function collectMessagesMatching(
+  socket: WebSocket, 
+  predicate: (m: unknown) => boolean, 
+  timeoutMs: number
+): Promise<unknown[]> {
+  return new Promise((resolve) => {
+    const messages: unknown[] = [];
+    const timeout = setTimeout(() => {
+      socket.off("message", onMessage);
+      resolve(messages);
+    }, timeoutMs);
+
+    const onMessage = (data: WebSocket.RawData, isBinary: boolean) => {
+      if (isBinary) return;
+      try {
+        const message = JSON.parse(data.toString()) as unknown;
+        if (predicate(message)) {
+          messages.push(message);
+        }
+      } catch {
+        // Ignore parsing errors
+      }
+    };
+    socket.on("message", onMessage);
+  });
+}
+
+// Track servers so we can force-close them in afterEach even if test fails
+let activeServers: VoiceWebSocketServer[] = [];
+let activeHttpServers: ReturnType<typeof createServer>[] = [];
+
+beforeEach(() => {
+  activeServers = [];
+  activeHttpServers = [];
+});
+
+afterEach(async () => {
+  for (const server of activeServers) {
+    await server.close().catch(() => undefined);
+  }
+  for (const httpServer of activeHttpServers) {
+    httpServer.close();
+  }
+  activeServers = [];
+  activeHttpServers = [];
+});
+
+describe("WT-03 Browser outbound pacing", () => {
+  it("browser adapter enqueues paced frames with consistent timing", async () => {
+    const httpServer = createServer();
+    activeHttpServers.push(httpServer);
+    
+    let session: VoiceAgentSession | null = null;
+    const server = await createVoiceWebSocketServer({
+      server: httpServer,
+      port: 0, // Let the system assign a port
+      createSession: () => {
+        session = new VoiceAgentSession({ plugins: {} });
+        return session;
+      },
+      outboundFrameDurationMs: 20,
+    });
+    activeServers.push(server);
+
+    const port = (server.address() as any).port;
+    const socket = await openBrowserSocketReady(browserUrl(port));
+
+    // Session is already started by the websocket server
+
+    // Send TTS audio - 160ms of 16kHz PCM16 (should be split into ~8 frames of 20ms each)
+    const sampleCount = Math.floor(0.16 * 16000); // 160ms at 16kHz
+    const audioSamples = new Int16Array(sampleCount).fill(1000); // Simple sine-like values
+    const audioBytes = pcm16SamplesToBytes(audioSamples);
+
+    // Collect tts_chunk messages over 200ms to ensure all frames arrive
+    const ttsChunksPromise = collectMessagesMatching(
+      socket,
+      (m: any) => m.type === "tts_chunk",
+      300
+    );
+
+    // Send TTS audio
+    session!.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "test-turn",
+      timestampMs: Date.now(),
+      audio: audioBytes,
+      sampleRateHz: 16000,
+    });
+
+    // End TTS
+    session!.bus.push(Route.Main, {
+      kind: "tts.end",
+      contextId: "test-turn",
+      timestampMs: Date.now(),
+    });
+
+    const ttsChunks = await ttsChunksPromise;
+    socket.close();
+
+    // Verify we got multiple frames (pacing is working)
+    expect(ttsChunks.length).toBeGreaterThan(1);
+    
+    // Verify each frame has expected structure
+    for (const chunk of ttsChunks) {
+      const c = chunk as any;
+      expect(c.type).toBe("tts_chunk");
+      expect(c.turnId).toBe("test-turn");
+      expect(c.sequence).toBeGreaterThan(0);
+      expect(c.sampleRateHz).toBe(16000);
+      expect(c.encoding).toBe("opus");
+      expect(c.channels).toBe(1);
+      expect(c.byteLength).toBeGreaterThan(0);
+      expect(c.durationMs).toBeGreaterThan(0);
+    }
+
+    // Verify sequence numbers increment
+    const sequences = ttsChunks.map((c: any) => c.sequence);
+    for (let i = 1; i < sequences.length; i++) {
+      expect(sequences[i]).toBe(sequences[i - 1] + 1);
+    }
+  });
+
+  it("emits tts.playout_progress events during paced playback", async () => {
+    const httpServer = createServer();
+    activeHttpServers.push(httpServer);
+    
+    let session: VoiceAgentSession | null = null;
+    const server = await createVoiceWebSocketServer({
+      server: httpServer,
+      port: 0,
+      createSession: () => {
+        session = new VoiceAgentSession({ plugins: {} });
+        return session;
+      },
+      outboundFrameDurationMs: 20,
+    });
+    activeServers.push(server);
+
+    const port = (server.address() as any).port;
+    const socket = await openBrowserSocketReady(browserUrl(port));
+
+    // Session is already started by the websocket server
+
+    // Track playout progress events
+    const progressEvents: any[] = [];
+    const removeProgressHandler = session!.bus.on("tts.playout_progress", (pkt: any) => {
+      if (pkt.contextId === "test-turn") {
+        progressEvents.push(pkt);
+      }
+    });
+
+    // Send TTS audio - 80ms worth
+    const sampleCount = Math.floor(0.08 * 16000);
+    const audioSamples = new Int16Array(sampleCount).fill(1000);
+    const audioBytes = pcm16SamplesToBytes(audioSamples);
+
+    session!.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "test-turn",
+      timestampMs: Date.now(),
+      audio: audioBytes,
+      sampleRateHz: 16000,
+    });
+
+    session!.bus.push(Route.Main, {
+      kind: "tts.end",
+      contextId: "test-turn",
+      timestampMs: Date.now(),
+    });
+
+    // Wait for playout to complete
+    await waitMs(150);
+
+    socket.close();
+    removeProgressHandler();
+
+    // Should have received playout progress events
+    expect(progressEvents.length).toBeGreaterThan(0);
+
+    // Last event should be completion
+    const lastEvent = progressEvents[progressEvents.length - 1];
+    expect(lastEvent.complete).toBe(true);
+  });
+
+  it("handles interrupt.tts by clearing playout queue and sending audio_clear", async () => {
+    const httpServer = createServer();
+    activeHttpServers.push(httpServer);
+    
+    let session: VoiceAgentSession | null = null;
+    const server = await createVoiceWebSocketServer({
+      server: httpServer,
+      port: 0,
+      createSession: () => {
+        session = new VoiceAgentSession({ plugins: {} });
+        return session;
+      },
+      outboundFrameDurationMs: 20,
+    });
+    activeServers.push(server);
+
+    const port = (server.address() as any).port;
+    const socket = await openBrowserSocketReady(browserUrl(port));
+
+    // Session is already started by the websocket server
+
+    // Send a longer TTS audio burst
+    const sampleCount = Math.floor(0.2 * 16000); // 200ms
+    const audioSamples = new Int16Array(sampleCount).fill(1000);
+    const audioBytes = pcm16SamplesToBytes(audioSamples);
+
+    session!.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "test-turn",
+      timestampMs: Date.now(),
+      audio: audioBytes,
+      sampleRateHz: 16000,
+    });
+
+    // Wait a bit for some frames to be queued
+    await waitMs(10);
+
+    // Listen for audio_clear message
+    const audioClearPromise = readJsonMatching(socket, (m: any) => m.type === "audio_clear");
+
+    // Send interrupt
+    session!.bus.push(Route.Main, {
+      kind: "interrupt.tts",
+      contextId: "test-turn",
+      timestampMs: Date.now(),
+    });
+
+    const audioClear = await audioClearPromise;
+    socket.close();
+
+    expect((audioClear as any).type).toBe("audio_clear");
+    expect((audioClear as any).turnId).toBe("test-turn");
+    expect((audioClear as any).reason).toBe("barge_in");
+  });
+
+  it("respects custom outbound frame duration and queue limits", async () => {
+    const httpServer = createServer();
+    activeHttpServers.push(httpServer);
+    
+    let session: VoiceAgentSession | null = null;
+    const server = await createVoiceWebSocketServer({
+      server: httpServer,
+      port: 0,
+      createSession: () => {
+        session = new VoiceAgentSession({ plugins: {} });
+        return session;
+      },
+      outboundFrameDurationMs: 40, // Larger frames
+      maxQueuedOutputAudioMs: 100, // Small queue to trigger overflow
+    });
+    activeServers.push(server);
+
+    const port = (server.address() as any).port;
+    const socket = await openBrowserSocketReady(browserUrl(port));
+
+    // Session is already started by the websocket server
+
+    // Send large audio burst to trigger overflow
+    const sampleCount = Math.floor(1.0 * 16000); // 1 second
+    const audioSamples = new Int16Array(sampleCount).fill(1000);
+    const audioBytes = pcm16SamplesToBytes(audioSamples);
+
+    const ttsChunksPromise = collectMessagesMatching(
+      socket,
+      (m: any) => m.type === "tts_chunk",
+      200
+    );
+
+    session!.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "test-turn",
+      timestampMs: Date.now(),
+      audio: audioBytes,
+      sampleRateHz: 16000,
+    });
+
+    const ttsChunks = await ttsChunksPromise;
+    socket.close();
+
+    // Tail-drop overflow: the head of the burst (up to the 100ms cap) still
+    // plays out — a tiny cap must never silence the stream entirely.
+    expect(ttsChunks.length).toBeGreaterThan(0);
+    expect(ttsChunks.length).toBeLessThan(20);
+    const firstChunk = ttsChunks[0] as any;
+    expect(firstChunk.durationMs).toBeGreaterThan(0);
+  });
+
+  it("delivers a long reply in full under the default playout bound (no overflow)", async () => {
+    const httpServer = createServer();
+    activeHttpServers.push(httpServer);
+
+    let session: VoiceAgentSession | null = null;
+    const metrics: Array<{ name: string; value: string }> = [];
+    const server = await createVoiceWebSocketServer({
+      server: httpServer,
+      port: 0,
+      createSession: () => {
+        session = new VoiceAgentSession({ plugins: {} });
+        return session;
+      },
+    });
+    activeServers.push(server);
+
+    const port = (server.address() as any).port;
+    const socket = await openBrowserSocketReady(browserUrl(port));
+    session!.bus.on("metric.conversation", (pkt) => {
+      const metric = pkt as unknown as { name: string; value: string };
+      metrics.push(metric);
+    });
+
+    const ttsChunksPromise = collectMessagesMatching(
+      socket,
+      (m: any) => m.type === "tts_chunk",
+      1_200, // collect for the full paced second
+    );
+    // A 1s burst — far beyond the old 200ms cap that used to clear-and-stop.
+    session!.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "default-bound-turn",
+      timestampMs: Date.now(),
+      audio: pcm16SamplesToBytes(new Int16Array(16000)),
+      sampleRateHz: 16000,
+    });
+
+    const ttsChunks = await ttsChunksPromise;
+    socket.close();
+
+    expect(ttsChunks.length).toBeGreaterThanOrEqual(40); // ≥800ms of the reply paced out
+    expect(metrics.some((m) => m.name.includes("overflow"))).toBe(false);
+  });
+
+  it("allows long playout queues only through an explicit override", async () => {
+    const httpServer = createServer();
+    activeHttpServers.push(httpServer);
+
+    let session: VoiceAgentSession | null = null;
+    const server = await createVoiceWebSocketServer({
+      server: httpServer,
+      port: 0,
+      createSession: () => {
+        session = new VoiceAgentSession({ plugins: {} });
+        return session;
+      },
+      maxQueuedOutputAudioMs: 1000,
+    });
+    activeServers.push(server);
+
+    const port = (server.address() as any).port;
+    const socket = await openBrowserSocketReady(browserUrl(port));
+    let closed = false;
+    socket.once("close", () => {
+      closed = true;
+    });
+
+    const ttsChunksPromise = collectMessagesMatching(
+      socket,
+      (m: any) => m.type === "tts_chunk",
+      150,
+    );
+    session!.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "explicit-long-queue-turn",
+      timestampMs: Date.now(),
+      audio: pcm16SamplesToBytes(new Int16Array(6400)),
+      sampleRateHz: 16000,
+    });
+
+    const chunks = await ttsChunksPromise;
+    socket.close();
+
+    expect(chunks.length).toBeGreaterThan(0);
+    expect(closed).toBe(false);
+  });
+});

package/src/edge-twilio.test.ts

@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+import {
+  PipelineBusImpl,
+  Route,
+  type UserAudioReceivedPacket,
+  type VoiceAgentSession,
+} from "@kuralle-syrinx/core";
+import {
+  decodeMuLawToPcm16,
+  encodePcm16ToMuLaw,
+  pcm16BytesToSamples,
+} from "@kuralle-syrinx/core/audio";
+import type { ManagedSocket, SocketData } from "@kuralle-syrinx/ws";
+import { InMemorySessionStore } from "./session-store.js";
+import { runTwilioEdgeWebSocketConnection } from "./edge-twilio.js";
+
+class FakeSocket implements ManagedSocket {
+  isOpen = true;
+  disposed = false;
+  readonly sent: SocketData[] = [];
+  #onMessage?: (data: SocketData, isBinary: boolean) => void;
+  #onClose?: (code: number, reason: string) => void;
+  get isOpenValue(): boolean {
+    return this.isOpen;
+  }
+  send(data: SocketData): void {
+    this.sent.push(data);
+  }
+  keepAlivePing(): void {}
+  async verify(): Promise<boolean> {
+    return this.isOpen;
+  }
+  dispose(): void {
+    this.disposed = true;
+    this.isOpen = false;
+    this.#onClose?.(1000, "disposed");
+  }
+  onOpen(): void {}
+  onMessage(handler: (data: SocketData, isBinary: boolean) => void): void {
+    this.#onMessage = handler;
+  }
+  onClose(handler: (code: number, reason: string) => void): void {
+    this.#onClose = handler;
+  }
+  onError(): void {}
+  emit(data: SocketData): void {
+    this.#onMessage?.(data, false);
+  }
+  json(): Array<Record<string, unknown>> {
+    return this.sent
+      .filter((entry): entry is string => typeof entry === "string")
+      .map((entry) => JSON.parse(entry) as Record<string, unknown>);
+  }
+}
+
+function fakeSession(received: UserAudioReceivedPacket[] = []): VoiceAgentSession {
+  const bus = new PipelineBusImpl();
+  bus.on("user.audio_received", (pkt) => {
+    received.push(pkt as UserAudioReceivedPacket);
+  });
+  return {
+    bus,
+    async start() {
+      void bus.start();
+    },
+    async close() {
+      bus.stop();
+    },
+    on() {},
+    off() {},
+    requestClientInterrupt() {},
+  } as unknown as VoiceAgentSession;
+}
+
+function bytesToBase64(bytes: Uint8Array): string {
+  return Buffer.from(bytes).toString("base64");
+}
+
+async function startConnection(received: UserAudioReceivedPacket[] = []) {
+  const socket = new FakeSocket();
+  const session = fakeSession(received);
+  await runTwilioEdgeWebSocketConnection(
+    socket,
+    new Request("https://edge.test/twilio?sessionId=tw1"),
+    {
+      sessionStore: new InMemorySessionStore(),
+      createSession: () => session,
+      keepAliveIntervalMs: 0,
+    },
+  );
+  socket.emit(JSON.stringify({ event: "connected", protocol: "Call" }));
+  socket.emit(JSON.stringify({
+    event: "start",
+    streamSid: "MZ123",
+    start: { streamSid: "MZ123", callSid: "CA456" },
+  }));
+  return { socket, session };
+}
+
+describe("Twilio edge ingress", () => {
+  it("decodes inbound mu-law media to 16k PCM user audio with a stable callSid context", async () => {
+    const received: UserAudioReceivedPacket[] = [];
+    const { socket } = await startConnection(received);
+
+    const samples8k = new Int16Array(160).fill(8000); // 20ms at 8k
+    const payload = bytesToBase64(encodePcm16ToMuLaw(samples8k));
+    socket.emit(JSON.stringify({ event: "media", streamSid: "MZ123", media: { payload } }));
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(received).toHaveLength(1);
+    expect(received[0]!.contextId).toBe("twilio-CA456");
+    const pcm = pcm16BytesToSamples(received[0]!.audio);
+    expect(pcm.length).toBe(320); // 8k → 16k doubles the samples
+    expect(Math.abs(pcm[100]! - 8000)).toBeLessThan(600); // mu-law quantization tolerance
+  });
+
+  it("encodes engine TTS PCM to 8k mu-law media frames for Twilio", async () => {
+    const { socket, session } = await startConnection();
+
+    const pcm16k = new Int16Array(640).fill(6000); // 40ms at 16k
+    const bytes = new Uint8Array(pcm16k.buffer.slice(0));
+    session.bus.push(Route.Critical, {
+      kind: "tts.audio",
+      contextId: "twilio-CA456",
+      timestampMs: Date.now(),
+      audio: bytes,
+      sampleRateHz: 16000,
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    const media = socket.json().filter((msg) => msg.event === "media");
+    expect(media).toHaveLength(1);
+    expect(media[0]!.streamSid).toBe("MZ123");
+    const payload = (media[0]!.media as { payload: string }).payload;
+    const decoded = decodeMuLawToPcm16(new Uint8Array(Buffer.from(payload, "base64")));
+    expect(decoded.length).toBe(320); // 16k → 8k halves the samples
+    expect(Math.abs(decoded[100]! - 6000)).toBeLessThan(600);
+  });
+
+  it("sends a clear event on interrupt.detected (barge-in)", async () => {
+    const { socket, session } = await startConnection();
+
+    session.bus.push(Route.Critical, {
+      kind: "interrupt.detected",
+      contextId: "twilio-CA456",
+      timestampMs: Date.now(),
+      source: "vad",
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(socket.json()).toContainEqual({ event: "clear", streamSid: "MZ123" });
+  });
+
+  it("releases the session when Twilio sends stop", async () => {
+    const { socket } = await startConnection();
+    socket.emit(JSON.stringify({ event: "stop", streamSid: "MZ123" }));
+    expect(socket.disposed).toBe(true);
+  });
+
+  it("rotates the uplink contextId after each completed turn", async () => {
+    const received: UserAudioReceivedPacket[] = [];
+    const { socket, session } = await startConnection(received);
+    const payload = bytesToBase64(encodePcm16ToMuLaw(new Int16Array(160).fill(4000)));
+
+    socket.emit(JSON.stringify({ event: "media", streamSid: "MZ123", media: { payload } }));
+    session.bus.push(Route.Main, {
+      kind: "eos.turn_complete",
+      contextId: "twilio-CA456",
+      timestampMs: Date.now(),
+      text: "first turn",
+      transcripts: [],
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    socket.emit(JSON.stringify({ event: "media", streamSid: "MZ123", media: { payload } }));
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(received).toHaveLength(2);
+    expect(received[0]!.contextId).toBe("twilio-CA456");
+    expect(received[1]!.contextId).toBe("twilio-CA456-t1");
+  });
+
+  it("buffers start/media that arrive before the session lease resolves", async () => {
+    const received: UserAudioReceivedPacket[] = [];
+    const socket = new FakeSocket();
+    const session = fakeSession(received);
+    const run = runTwilioEdgeWebSocketConnection(
+      socket,
+      new Request("https://edge.test/twilio?sessionId=tw-early"),
+      {
+        sessionStore: new InMemorySessionStore(),
+        createSession: async () => {
+          await new Promise((resolve) => setTimeout(resolve, 30)); // slow startup
+          return session;
+        },
+        keepAliveIntervalMs: 0,
+      },
+    );
+    // Twilio talks immediately — before startup finishes.
+    socket.emit(JSON.stringify({ event: "connected", protocol: "Call" }));
+    socket.emit(JSON.stringify({
+      event: "start",
+      streamSid: "MZ9",
+      start: { streamSid: "MZ9", callSid: "CA9" },
+    }));
+    const payload = bytesToBase64(encodePcm16ToMuLaw(new Int16Array(160).fill(4000)));
+    socket.emit(JSON.stringify({ event: "media", streamSid: "MZ9", media: { payload } }));
+    await run;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(received).toHaveLength(1);
+    expect(received[0]!.contextId).toBe("twilio-CA9");
+  });
+});