@kuralle-syrinx/server-websocket 2.1.1 → 3.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/server-websocket",
-  "version": "2.1.1",
+  "version": "3.1.0",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -15,8 +15,8 @@
   "dependencies": {
     "@evan/opus": "1.0.3",
     "ws": "^8.18.0",
-    "@kuralle-syrinx/core": "2.1.1",
-    "@kuralle-syrinx/ws": "2.1.1"
+    "@kuralle-syrinx/core": "3.1.0",
+    "@kuralle-syrinx/ws": "3.1.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

package/src/edge.test.ts

@@ -13,7 +13,7 @@ import {
 import { pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
 import type { ManagedSocket, SocketData } from "@kuralle-syrinx/ws";
 import { InMemorySessionStore } from "./session-store.js";
-import { runVoiceEdgeWebSocketConnection } from "./edge.js";
+import { runVoiceEdgeWebSocketConnection, type EdgeRecorder } from "./edge.js";
 
 const KEEP_ALIVE_KEY = "voice.edge.keep_alive";
 
@@ -39,6 +39,7 @@ class FakeSocket implements ManagedSocket {
   readonly sent: SocketData[] = [];
   #onMessage?: (data: SocketData, isBinary: boolean) => void;
   #onClose?: (code: number, reason: string) => void;
+  #onError?: (err: Error) => void;
   get isOpenValue(): boolean {
     return this.isOpen;
   }
@@ -61,10 +62,15 @@ class FakeSocket implements ManagedSocket {
   onClose(handler: (code: number, reason: string) => void): void {
     this.#onClose = handler;
   }
-  onError(): void {}
+  onError(handler: (err: Error) => void): void {
+    this.#onError = handler;
+  }
   emit(data: SocketData, isBinary = false): void {
     this.#onMessage?.(data, isBinary);
   }
+  emitError(err: Error = new Error("socket error")): void {
+    this.#onError?.(err);
+  }
   json(): Array<Record<string, unknown>> {
     return this.sent
       .filter((s): s is string => typeof s === "string")
@@ -246,6 +252,110 @@ describe("edge inbound binary audio envelopes", () => {
   });
 });
 
+describe("edge inbound JSON audio (sample-rate handling)", () => {
+  function sessionCapturing(
+    audio: UserAudioReceivedPacket[],
+    turns: Array<{ contextId: string; previousContextId?: string }>,
+  ): VoiceAgentSession {
+    const bus = new PipelineBusImpl();
+    bus.on("user.audio_received", (p) => {
+      audio.push(p as UserAudioReceivedPacket);
+    });
+    bus.on("turn.change", (p) => {
+      turns.push(p as { contextId: string; previousContextId?: string });
+    });
+    return {
+      bus,
+      async start() {
+        void bus.start();
+      },
+      async close() {
+        bus.stop();
+      },
+      on() {},
+      off() {},
+      requestClientInterrupt() {},
+    } as unknown as VoiceAgentSession;
+  }
+
+  it("resamples JSON audio from the client sampleRateHz to the engine input rate", async () => {
+    const audio: UserAudioReceivedPacket[] = [];
+    const turns: Array<{ contextId: string; previousContextId?: string }> = [];
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => sessionCapturing(audio, turns),
+    });
+    waitForReady(socket);
+
+    // 8 kHz mono PCM16 → engine default 16 kHz: sample count (and byte length) should ~double.
+    const pcm8k = pcm16SamplesToBytes(new Int16Array(80).fill(1000));
+    socket.emit(
+      JSON.stringify({ type: "audio", audio: Buffer.from(pcm8k).toString("base64"), sampleRateHz: 8000, contextId: "json-turn" }),
+    );
+    await new Promise((r) => setTimeout(r, 20));
+
+    expect(audio).toHaveLength(1);
+    // BUG: JSON path pushed raw 8 kHz bytes unchanged instead of resampling to 16 kHz.
+    expect(audio[0]!.audio.byteLength).toBeGreaterThan(pcm8k.byteLength * 1.5);
+    // BUG: JSON path never emitted turn.change on contextId rotation (text/binary branches do).
+    expect(turns.some((t) => t.contextId === "json-turn")).toBe(true);
+  });
+});
+
+describe("edge recorder finalize on disconnect", () => {
+  function fakeRecorder() {
+    const calls: Array<{ sessionId: string; closedAtMs: number }> = [];
+    const rec: EdgeRecorder = {
+      onUserAudio() {},
+      onAssistantAudio() {},
+      finalize(meta) {
+        calls.push(meta);
+      },
+    };
+    return { rec, calls };
+  }
+
+  it("finalizes the recording on an error-path disconnect, not just a clean close", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    const { rec, calls } = fakeRecorder();
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => fakeSession(),
+      recorder: rec,
+    });
+    waitForReady(socket);
+
+    socket.emitError(new Error("abnormal disconnect")); // error path, no clean onClose
+    await new Promise((resolve) => setTimeout(resolve, 0));
+
+    expect(calls).toHaveLength(1); // recording must still be finalized
+  });
+
+  it("finalizes exactly once when both an error and a close fire", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    const { rec, calls } = fakeRecorder();
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => fakeSession(),
+      recorder: rec,
+    });
+    waitForReady(socket);
+
+    socket.emitError(new Error("drop"));
+    socket.dispose(); // fires onClose
+    await new Promise((resolve) => setTimeout(resolve, 0));
+
+    expect(calls).toHaveLength(1); // idempotent teardown — not twice
+  });
+});
+
 describe("edge barge-in downlink", () => {
   it("sends audio_clear and agent_interrupted when an interrupt is detected", async () => {
     const socket = new FakeSocket();
@@ -270,3 +380,73 @@ describe("edge barge-in downlink", () => {
     expect(socket.json()).toContainEqual({ type: "agent_interrupted", turnId: "assistant-turn", reason: "barge_in" });
   });
 });
+
+describe("edge client playout progress", () => {
+  interface ProgressPacket {
+    contextId: string;
+    playedOutMs: number;
+    complete: boolean;
+  }
+
+  function sessionCapturingProgress(progress: ProgressPacket[]): VoiceAgentSession {
+    const bus = new PipelineBusImpl();
+    bus.on("tts.playout_progress", (p) => {
+      progress.push(p as unknown as ProgressPacket);
+    });
+    return {
+      bus,
+      async start() {
+        void bus.start();
+      },
+      async close() {
+        bus.stop();
+      },
+      on() {},
+      off() {},
+      requestClientInterrupt() {},
+    } as unknown as VoiceAgentSession;
+  }
+
+  it("maps a client playout_progress message onto a tts.playout_progress bus packet", async () => {
+    const progress: ProgressPacket[] = [];
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => sessionCapturingProgress(progress),
+    });
+    waitForReady(socket);
+
+    socket.emit(
+      JSON.stringify({ type: "playout_progress", contextId: "assistant-turn", playedOutMs: 1280 }),
+    );
+    socket.emit(
+      JSON.stringify({ type: "playout_progress", contextId: "assistant-turn", playedOutMs: 2400, complete: true }),
+    );
+    await new Promise((r) => setTimeout(r, 20));
+
+    expect(progress).toEqual([
+      { kind: "tts.playout_progress", contextId: "assistant-turn", playedOutMs: 1280, complete: false, timestampMs: expect.any(Number) },
+      { kind: "tts.playout_progress", contextId: "assistant-turn", playedOutMs: 2400, complete: true, timestampMs: expect.any(Number) },
+    ]);
+  });
+
+  it("rejects a playout_progress message without a numeric playedOutMs", async () => {
+    const progress: ProgressPacket[] = [];
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => sessionCapturingProgress(progress),
+    });
+    waitForReady(socket);
+
+    socket.emit(JSON.stringify({ type: "playout_progress", contextId: "assistant-turn" }));
+    await new Promise((r) => setTimeout(r, 20));
+
+    expect(progress).toHaveLength(0);
+    expect(socket.json().some((m) => m.type === "error" && m.category === "invalid_input")).toBe(true);
+  });
+});

package/src/edge.ts

@@ -6,6 +6,7 @@ import {
   encodeSyrinxAudioEnvelope,
   type TextToSpeechAudioPacket,
   type TextToSpeechEndPacket,
+  type TextToSpeechPlayoutProgressPacket,
   type UserAudioReceivedPacket,
   type UserTextReceivedPacket,
   type VoiceAgentSession,
@@ -88,6 +89,7 @@ type ClientMessage =
   | { readonly type: "text"; readonly text: string; readonly contextId?: string }
   | { readonly type: "audio"; readonly audio: string; readonly contextId?: string; readonly sampleRateHz: number; readonly sequence?: number }
   | { readonly type: "client_interrupt"; readonly assistantContextId?: string; readonly contextId?: string }
+  | { readonly type: "playout_progress"; readonly contextId?: string; readonly playedOutMs: number; readonly complete?: boolean }
   | { readonly type: "codec_capability"; readonly downlinkEncoding: "pcm_s16le" | "opus" }
   | { readonly type: "ping" };
 
@@ -207,7 +209,12 @@ export async function runVoiceEdgeWebSocketConnection(
     }
   });
 
-  socket.onClose(() => {
+  // Tear down on EITHER a clean close or an error-path disconnect, exactly once (guarded by
+  // `closed`). On Cloudflare Workers a DO webSocketError surfaces here as onError with no
+  // matching onClose, so finalize/release must also run on the error path — otherwise the
+  // in-memory recording is lost and the session lease never releases.
+  const teardownConnection = (): void => {
+    if (closed) return;
     closed = true;
     for (const dispose of disposers.splice(0)) dispose();
     if (state.managed) {
@@ -219,7 +226,9 @@ export async function runVoiceEdgeWebSocketConnection(
         options.recorder.finalize({ sessionId: state.managed?.id ?? "unknown", closedAtMs: Date.now() }),
       ).catch(() => undefined);
     }
-  });
+  };
+  socket.onClose(teardownConnection);
+  socket.onError(teardownConnection);
 
   try {
     const requestedSessionId = sanitizeSessionId(sessionIdFromRequest(request) ?? sessionIdFn(request));
@@ -460,6 +469,23 @@ function handleClientMessage(
     if (interruptedContextId) session.requestClientInterrupt(interruptedContextId);
     return currentContextId;
   }
+  if (message.type === "playout_progress") {
+    // On client-rendered-audio transports the browser is the playout clock: the server streams
+    // audio envelopes and the client schedules them. Map its reported position onto the same
+    // `tts.playout_progress` packet the server-paced path emits, so turn-truncation (realtime
+    // barge-in) and turn-metrics consume an accurate played-out offset instead of a stale 0.
+    const progressContextId = message.contextId ?? currentContextId;
+    if (progressContextId) {
+      session.bus.push(Route.Main, {
+        kind: "tts.playout_progress",
+        contextId: progressContextId,
+        timestampMs: Date.now(),
+        playedOutMs: message.playedOutMs,
+        complete: message.complete ?? false,
+      } satisfies TextToSpeechPlayoutProgressPacket);
+    }
+    return currentContextId;
+  }
   if (message.type === "text") {
     const nextContextId = message.contextId ?? contextId();
     session.bus.push(Route.Main, {
@@ -478,12 +504,23 @@ function handleClientMessage(
     return nextContextId;
   }
   const nextContextId = message.contextId ?? currentContextId;
+  if (nextContextId !== currentContextId) {
+    session.bus.push(Route.Main, {
+      kind: "turn.change",
+      contextId: nextContextId,
+      previousContextId: currentContextId,
+      reason: "websocket_audio_turn",
+      timestampMs: Date.now(),
+    });
+  }
+  rememberContextSampleRate(contextSampleRates, nextContextId, message.sampleRateHz);
   rememberInputSequence(session, inputSequence, nextContextId, message.sequence);
+  const audio = resampleAudioBytes(decodeBase64(message.audio), message.sampleRateHz, inputSampleRateHz, streamingResamplers);
   session.bus.push(Route.Main, {
     kind: "user.audio_received",
     contextId: nextContextId,
     timestampMs: Date.now(),
-    audio: decodeBase64(message.audio),
+    audio,
   } satisfies UserAudioReceivedPacket);
   return nextContextId;
 }
@@ -504,6 +541,18 @@ function parseClientMessage(value: unknown): ClientMessage {
       contextId: optionalString(value.contextId),
     };
   }
+  if (value.type === "playout_progress") {
+    const playedOutMs = nonNegativeInteger(value.playedOutMs);
+    if (playedOutMs === null) {
+      throw new Error("Websocket playout_progress playedOutMs must be a non-negative integer");
+    }
+    return {
+      type: "playout_progress",
+      contextId: optionalString(value.contextId),
+      playedOutMs,
+      complete: typeof value.complete === "boolean" ? value.complete : undefined,
+    };
+  }
   if (value.type === "text") {
     const text = optionalString(value.text);
     if (!text) throw new Error("Websocket JSON text must be a non-empty string");

package/src/telnyx.test.ts

@@ -848,6 +848,50 @@ describe("createTelnyxMediaStreamServer", () => {
     await server.close();
   });
 
+  it("attributes the final paced frame to its contextId (playout clock not short by one frame)", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const server = registerServer(await createTelnyxMediaStreamServer({
+      port: 0,
+      outputSampleRateHz: 16000,
+      createSession: () => session,
+    }));
+    const address = server.address();
+    if (!address || typeof address === "string") throw new Error("Expected TCP address");
+
+    const progress: Array<{ playedOutMs: number; complete: boolean }> = [];
+    session.bus.on("tts.playout_progress", (pkt) => {
+      const p = pkt as TextToSpeechPlayoutProgressPacket;
+      if (p.contextId === "telnyx-final-frame") progress.push({ playedOutMs: p.playedOutMs, complete: p.complete });
+    });
+
+    const client = await openSocket(telnyxUrl(address.port));
+    client.send(JSON.stringify(telnyxStart()));
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    // 200ms @ 16 kHz = exactly 10 paced frames of 20ms; the playout clock must count all 10.
+    const samples = new Int16Array(3200);
+    for (let i = 0; i < samples.length; i += 1) samples[i] = i % 2 === 0 ? 1200 : -1200;
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "telnyx-final-frame",
+      timestampMs: Date.now(),
+      audio: pcm16SamplesToBytes(samples),
+      sampleRateHz: 16000,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.end",
+      contextId: "telnyx-final-frame",
+      timestampMs: Date.now(),
+    });
+    await waitForCondition(() => progress.at(-1)?.complete === true);
+
+    // BUG: the final frame was rebuilt without contextId, so its 20ms was dropped (reported 180, not 200).
+    expect(progress.at(-1)?.playedOutMs).toBe(200);
+
+    client.close();
+    await server.close();
+  });
+
   it("uses configured L16 bidirectional output rather than assuming the inbound codec", async () => {
     const session = new VoiceAgentSession({ plugins: {} });
     const server = registerServer(await createTelnyxMediaStreamServer({

package/src/telnyx.ts

@@ -241,6 +241,7 @@ export async function createTelnyxMediaStreamServer(
             const finalFrame = frames.at(-1);
             if (finalFrame) {
               frames[frames.length - 1] = {
+                contextId,
                 send: finalFrame.send,
                 afterSend: () => {
                   if (state.stopped) return;