@kuralle-syrinx/server-websocket 3.1.0 → 4.1.0

package/src/edge-twilio.ts

@@ -26,11 +26,16 @@ import {
   type StreamingPcm16Resampler,
 } from "@kuralle-syrinx/core/audio";
 import type { ManagedSocket, SocketData } from "@kuralle-syrinx/ws";
+import {
+  BackgroundAudioMixer,
+  wireBackgroundThinking,
+  type BackgroundAudioConfig,
+} from "./background-audio.js";
 import {
   createWorkersInboundSocket,
   type WorkersDurableObjectWebSocketContext,
 } from "@kuralle-syrinx/ws/workers";
-import type { SessionStore } from "./session-store.js";
+import type { SessionStore, ManagedSession } from "./session-store.js";
 
 const TWILIO_SAMPLE_RATE_HZ = 8_000;
 const DEFAULT_RESUME_WINDOW_MS = 15_000;
@@ -45,6 +50,14 @@ export interface TwilioEdgeWebSocketOptions {
   readonly scheduler?: Scheduler;
   /** Engine-side PCM rate (default 16000). Twilio's leg is always 8 kHz μ-law. */
   readonly engineSampleRateHz?: number;
+  /**
+   * Ambient/thinking bed: mixed (ducked) under assistant speech, and sent as
+   * comfort-noise frames between turns — pure digital silence on a phone line
+   * reads as "the call died". Thinking loop follows the G3 tool-call cues.
+   */
+  readonly backgroundAudio?: BackgroundAudioConfig;
+  /** Cadence of idle comfort-noise frames (ms). @default 200 */
+  readonly backgroundIdleFrameMs?: number;
   readonly resumeWindowMs?: number;
   readonly keepAliveIntervalMs?: number;
   readonly idleTimeoutMs?: number;
@@ -82,6 +95,8 @@ export async function runTwilioEdgeWebSocketConnection(
   const downlinkResamplers = new Map<string, StreamingPcm16Resampler>();
   const disposers: Array<() => void> = [];
   let session: VoiceAgentSession | null = null;
+  let managed: ManagedSession | null = null;
+  let pendingLease: Promise<{ managed: ManagedSession }> | null = null;
   let sessionId = "";
   let streamSid = "";
   let contextId = "";
@@ -100,9 +115,26 @@ export async function runTwilioEdgeWebSocketConnection(
     if (closed) return;
     closed = true;
     scheduler.cancel(KEEP_ALIVE_KEY);
+    scheduler.cancel("voice.edge.twilio.startup");
     for (const dispose of disposers.splice(0)) dispose();
-    if (session && sessionId) {
+    if (managed && sessionId) {
+      // Decrement the connection count BEFORE releasing (R1) — otherwise release
+      // early-returns while connectionCount > 0 and session.close() never runs, so
+      // Deepgram/TTS provider sockets + the reasoner leak until DO eviction. A
+      // caller-hangup (`stopped`) releases immediately (retain 0); a transient drop
+      // keeps the session warm for the resume window.
+      managed.connectionCount = Math.max(0, managed.connectionCount - 1);
       void options.sessionStore.release(sessionId, stopped ? 0 : resumeWindowMs);
+    } else if (pendingLease && sessionId) {
+      // Closed before the lease was adopted (e.g. startup-timeout race): tear the
+      // in-flight session down when it resolves so it is never orphaned in the store.
+      const id = sessionId;
+      void pendingLease
+        .then((leased) => {
+          leased.managed.connectionCount = Math.max(0, leased.managed.connectionCount - 1);
+          return options.sessionStore.release(id, 0);
+        })
+        .catch(() => undefined);
     }
   };
 
@@ -141,37 +173,61 @@ export async function runTwilioEdgeWebSocketConnection(
         reject(new Error("twilio session startup timeout"));
       });
     });
-    const leased = await Promise.race([
-      options.sessionStore.lease(sessionId, async () => {
-        const sess = await options.createSession(request);
-        await sess.start();
-        return {
-          id: sessionId,
-          session: sess,
-          currentContextId: "",
-          contextSampleRates: new Map(),
-          inputSequence: { lastSequence: null },
-          turnMetricsTurns: new Map(),
-          closeTimer: null,
-          connectionCount: 1,
-        };
-      }),
-      startupTimer,
-    ]);
+    pendingLease = options.sessionStore.lease(sessionId, async () => {
+      const sess = await options.createSession(request);
+      await sess.start();
+      return {
+        id: sessionId,
+        session: sess,
+        currentContextId: "",
+        contextSampleRates: new Map(),
+        inputSequence: { lastSequence: null },
+        turnMetricsTurns: new Map(),
+        closeTimer: null,
+        connectionCount: 1,
+      };
+    });
+    const leased = await Promise.race([pendingLease, startupTimer]);
+    pendingLease = null;
     scheduler.cancel("voice.edge.twilio.startup");
-    session = leased.managed.session;
+    managed = leased.managed;
+    session = managed.session;
     if (closed) {
+      // The socket closed while the session was starting up — tear the just-started
+      // session down instead of orphaning it (decrement so release actually closes it).
+      managed.connectionCount = Math.max(0, managed.connectionCount - 1);
       await options.sessionStore.release(sessionId, 0);
       return;
     }
 
+    // Background bed: ducked under speech in the tts.audio handler below, and
+    // sent as idle comfort-noise frames between turns — a phone line carrying
+    // pure digital silence reads as "the call died". Twilio's `clear` on
+    // barge-in drops any queued bed audio; the ticker refills on the next tick.
+    const backgroundAudio = options.backgroundAudio
+      ? new BackgroundAudioMixer(options.backgroundAudio)
+      : null;
+    if (backgroundAudio) {
+      wireBackgroundThinking(session, backgroundAudio);
+      const idleFrameMs = options.backgroundIdleFrameMs ?? 200;
+      const idleTimer = setInterval(() => {
+        if (!streamSid || stopped || closed || !socket.isOpen) return;
+        const frame = backgroundAudio.idleFrame(idleFrameMs, TWILIO_SAMPLE_RATE_HZ);
+        if (!frame) return;
+        const mulaw = encodePcm16ToMuLaw(pcm16BytesToSamples(frame));
+        sendTwilioJson({ event: "media", streamSid, media: { payload: bytesToBase64(mulaw) } });
+      }, idleFrameMs);
+      disposers.push(() => clearInterval(idleTimer));
+    }
+
     // Downlink: engine PCM → 8 kHz μ-law media frames; barge-in → clear.
     disposers.push(
       session.bus.on("tts.audio", (pkt) => {
         if (!streamSid) return;
         const audio = pkt as TextToSpeechAudioPacket;
-        const samples = pcm16BytesToSamples(audio.audio);
         const sourceRate = audio.sampleRateHz ?? engineRate;
+        const wireAudio = backgroundAudio ? backgroundAudio.mix(audio.audio, sourceRate) : audio.audio;
+        const samples = pcm16BytesToSamples(wireAudio);
         const resampled = resamplePcm16Streaming(downlinkResamplers, samples, sourceRate, TWILIO_SAMPLE_RATE_HZ);
         const mulaw = encodePcm16ToMuLaw(resampled);
         sendTwilioJson({

package/src/edge.test.ts

@@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest";
 import {
   PipelineBusImpl,
   Route,
+  VoiceAgentSession as VoiceAgentSessionImpl,
   encodeSyrinxAudioEnvelope,
   type Scheduler,
   type ScheduledCallback,
@@ -381,6 +382,84 @@ describe("edge barge-in downlink", () => {
   });
 });
 
+describe("edge tool-call cues (G3/WBS-3)", () => {
+  it("surfaces the typed lifecycle as tool_call_* wire messages: started → delayed → complete", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    const session = new VoiceAgentSessionImpl({ plugins: {}, delayCueAfterMs: 20 });
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => session,
+    });
+    waitForReady(socket);
+
+    session.bus.push(Route.Main, {
+      kind: "llm.tool_call",
+      contextId: "turn-1",
+      timestampMs: Date.now(),
+      toolId: "t1",
+      toolName: "consult_knowledge",
+      toolArgs: { query: "fees" },
+    });
+    await new Promise((resolve) => setTimeout(resolve, 50));
+    session.bus.push(Route.Main, {
+      kind: "llm.tool_result",
+      contextId: "turn-1",
+      timestampMs: Date.now(),
+      toolId: "t1",
+      toolName: "consult_knowledge",
+      result: "answer",
+    });
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(socket.json()).toContainEqual({
+      type: "tool_call_started", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge",
+    });
+    expect(socket.json()).toContainEqual({
+      type: "tool_call_delayed", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge", afterMs: 20,
+    });
+    expect(socket.json()).toContainEqual({
+      type: "tool_call_complete", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge",
+    });
+  });
+
+  it("surfaces tool_call_failed when a barge-in aborts the pending delegate (R5)", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    const session = new VoiceAgentSessionImpl({ plugins: {}, delayCueAfterMs: 0 });
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => session,
+    });
+    waitForReady(socket);
+
+    session.bus.push(Route.Main, {
+      kind: "llm.tool_call",
+      contextId: "turn-1",
+      timestampMs: Date.now(),
+      toolId: "t1",
+      toolName: "consult_knowledge",
+      toolArgs: {},
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    session.bus.push(Route.Critical, {
+      kind: "interrupt.detected",
+      contextId: "turn-1",
+      timestampMs: Date.now(),
+      source: "vad",
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(socket.json()).toContainEqual({
+      type: "tool_call_failed", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge",
+    });
+    // Barge-in downlink unaffected — the flush messages still went out.
+    expect(socket.json().some((m) => m.type === "audio_clear")).toBe(true);
+  });
+});
+
 describe("edge client playout progress", () => {
   interface ProgressPacket {
     contextId: string;
@@ -450,3 +529,63 @@ describe("edge client playout progress", () => {
     expect(socket.json().some((m) => m.type === "error" && m.category === "invalid_input")).toBe(true);
   });
 });
+
+describe("edge background audio", () => {
+  it("mixes the bed into the wire audio while the recorder keeps the clean track", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    const recorded: Int16Array[] = [];
+    const rec: EdgeRecorder = {
+      onUserAudio() {},
+      onAssistantAudio(_contextId: string, audio: Uint8Array) {
+        recorded.push(new Int16Array(audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength)));
+      },
+      finalize() {},
+    };
+    const bus = new PipelineBusImpl();
+    const session = {
+      bus,
+      async start() {
+        void bus.start();
+      },
+      async close() {
+        bus.stop();
+      },
+      on() {},
+      off() {},
+      requestClientInterrupt() {},
+    } as unknown as VoiceAgentSession;
+
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => session,
+      recorder: rec,
+      backgroundAudio: {
+        ambient: { pcm: new Int16Array(160).fill(2000), sampleRateHz: 16000, gain: 0.5 },
+        duckWhileSpeaking: 0.5,
+        fadeMs: 0,
+      },
+    });
+    waitForReady(socket);
+
+    bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "bg-turn",
+      timestampMs: Date.now(),
+      audio: pcm16SamplesToBytes(new Int16Array(4)), // silent speech chunk
+      sampleRateHz: 16000,
+    });
+    await new Promise((r) => setTimeout(r, 20));
+
+    // Recorder saw the CLEAN chunk.
+    expect(recorded).toHaveLength(1);
+    expect(Array.from(recorded[0]!)).toEqual([0, 0, 0, 0]);
+
+    // The wire envelope carries the bed: 2000 × 0.5 gain × 0.5 duck = 500.
+    const binary = socket.sent.find((d): d is Uint8Array => d instanceof Uint8Array);
+    expect(binary).toBeDefined();
+    const wireSamples = new Int16Array(binary!.buffer.slice(binary!.byteOffset + (binary!.byteLength - 8)));
+    expect(Array.from(wireSamples)).toEqual([500, 500, 500, 500]);
+  });
+});

package/src/edge.ts

@@ -14,6 +14,12 @@ import {
 } from "@kuralle-syrinx/core";
 import { TimerScheduler, type Scheduler } from "@kuralle-syrinx/core";
 import { type StreamingPcm16Resampler } from "@kuralle-syrinx/core/audio";
+import {
+  BackgroundAudioMixer,
+  wireBackgroundThinking,
+  type BackgroundAudioConfig,
+} from "./background-audio.js";
+export type { BackgroundAudioConfig, BackgroundAudioSource } from "./background-audio.js";
 import {
   createWorkersInboundSocket,
   type WorkersDurableObjectWebSocketContext,
@@ -76,6 +82,12 @@ export interface VoiceEdgeWebSocketOptions {
    * the Syrinx binary envelope instead.
    */
   readonly rawBinaryInput?: boolean;
+  /**
+   * Ambient/thinking bed mixed (ducked) under assistant speech. Browser edge
+   * sends no idle bed between turns — a web client can loop ambience locally;
+   * the server-side bed exists for wires with no client runtime (telephony).
+   */
+  readonly backgroundAudio?: BackgroundAudioConfig;
   readonly sessionStore: SessionStore;
   readonly scheduler?: Scheduler;
 }
@@ -265,7 +277,14 @@ export async function runVoiceEdgeWebSocketConnection(
       await options.sessionStore.release(leased.managed.id, 0);
       return;
     }
-    wireEdgeSessionEvents(session, socket, disposers, outputSampleRateHz, options.recorder);
+    wireEdgeSessionEvents(
+      session,
+      socket,
+      disposers,
+      outputSampleRateHz,
+      options.recorder,
+      options.backgroundAudio ? new BackgroundAudioMixer(options.backgroundAudio) : undefined,
+    );
     if (options.recorder) {
       const recorder = options.recorder;
       disposers.push(
@@ -349,7 +368,9 @@ function wireEdgeSessionEvents(
   disposers: Array<() => void>,
   outputSampleRateHz: number,
   recorder?: EdgeRecorder,
+  backgroundAudio?: BackgroundAudioMixer,
 ): void {
+  if (backgroundAudio) wireBackgroundThinking(session, backgroundAudio);
   const onSession = <K extends keyof VoiceAgentSessionEvents>(
     event: K,
     handler: VoiceAgentSessionEvents[K],
@@ -367,6 +388,19 @@ function wireEdgeSessionEvents(
   onSession("agent_text_delta", (event) => {
     sendJson(socket, { type: "agent_chunk", turnId: event.turnId, text: event.delta });
   });
+  // G3 (RFC bimodel-delegate-seam): typed preamble/filler lifecycle. The standard
+  // "thinking" wire cue — clients key earcons/indicators on these instead of an
+  // app-invented message. started fires before the reasoner runs; delayed is the
+  // time-triggered "still working"; complete/failed end the wait (incl. barge-in).
+  onSession("tool_call_cue", (event) => {
+    sendJson(socket, {
+      type: `tool_call_${event.phase}`,
+      turnId: event.turnId,
+      toolId: event.toolId,
+      toolName: event.toolName,
+      ...(event.afterMs !== undefined ? { afterMs: event.afterMs } : {}),
+    });
+  });
   onSession("agent_finished", (event) => {
     sendJson(socket, { type: "agent_end", turnId: event.turnId });
   });
@@ -378,27 +412,29 @@ function wireEdgeSessionEvents(
     session.bus.on("tts.audio", (pkt) => {
       const audio = pkt as TextToSpeechAudioPacket;
       const ttsSampleRate = requireTtsAudioSampleRate(audio.sampleRateHz) ?? outputSampleRateHz;
+      // Recorder gets the CLEAN assistant track; the bed is a wire-level effect.
       recorder?.onAssistantAudio(audio.contextId, audio.audio, ttsSampleRate);
+      const wireAudio = backgroundAudio ? backgroundAudio.mix(audio.audio, ttsSampleRate) : audio.audio;
       sendJson(socket, {
         type: "tts_chunk",
         turnId: audio.contextId,
         sequence: 1,
-        sampleRateHz: requireTtsAudioSampleRate(audio.sampleRateHz) ?? outputSampleRateHz,
+        sampleRateHz: ttsSampleRate,
         encoding: "pcm_s16le",
         channels: 1,
-        byteLength: audio.audio.byteLength,
-        durationMs: pcm16DurationMs(audio.audio, outputSampleRateHz),
+        byteLength: wireAudio.byteLength,
+        durationMs: pcm16DurationMs(wireAudio, outputSampleRateHz),
       });
       socket.send(encodeSyrinxAudioEnvelope({
         type: "audio",
         contextId: audio.contextId,
         sequence: 1,
-        sampleRateHz: requireTtsAudioSampleRate(audio.sampleRateHz) ?? outputSampleRateHz,
+        sampleRateHz: ttsSampleRate,
         encoding: "pcm_s16le",
         channels: 1,
-        byteLength: audio.audio.byteLength,
-        durationMs: pcm16DurationMs(audio.audio, outputSampleRateHz),
-      }, audio.audio));
+        byteLength: wireAudio.byteLength,
+        durationMs: pcm16DurationMs(wireAudio, outputSampleRateHz),
+      }, wireAudio));
     }),
     session.bus.on("tts.end", (pkt) => {
       const end = pkt as TextToSpeechEndPacket;

package/src/inbound-audio.test.ts

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+import { encodeSyrinxAudioEnvelope } from "@kuralle-syrinx/core";
+import { decodeInboundBinaryAudio, resampleAudioBytes, type OpusIngressDecoder } from "./inbound-audio.js";
+
+// Regression for the opus mic-uplink double-resample P0: opus ingress is decoded
+// AND resampled to the engine rate inside decodeInboundBinaryAudio, so it must
+// report the ENGINE rate — not the 48 kHz header rate — or the caller resamples
+// the already-16 kHz audio a second time and delivers ~1/3 the samples (3× fast).
+describe("decodeInboundBinaryAudio opus rate", () => {
+  const ENGINE_RATE = 16000;
+  const WIRE_RATE = 48000;
+
+  // A fake opus decoder returning PCM at the opus native rate (48 kHz), as the real
+  // decoder does. decodeInboundBinaryAudio then resamples it ONCE to the engine rate.
+  // 20 ms at 48 kHz mono PCM16 = 960 samples = 1920 bytes → 320 samples = 640 bytes @ 16 kHz.
+  const opusNativePcm = new Uint8Array(1920);
+  const fakeOpusDecoder: OpusIngressDecoder = () => opusNativePcm;
+
+  function opusEnvelope(): Uint8Array {
+    // The opus payload bytes are opaque to the decoder mock.
+    const opusPayload = new Uint8Array([1, 2, 3, 4]);
+    return encodeSyrinxAudioEnvelope(
+      {
+        type: "audio",
+        contextId: "turn-1",
+        sampleRateHz: WIRE_RATE,
+        sequence: 1,
+        encoding: "opus",
+        channels: 1,
+        byteLength: opusPayload.byteLength,
+        durationMs: 20,
+      },
+      opusPayload,
+    );
+  }
+
+  it("reports the engine rate, not the header rate, so the caller does not resample again", () => {
+    const decoded = decodeInboundBinaryAudio(
+      opusEnvelope(),
+      ENGINE_RATE,
+      false,
+      ENGINE_RATE,
+      new Map(),
+      fakeOpusDecoder,
+    );
+
+    // Decoded once from 48 kHz → 16 kHz inside decodeInboundBinaryAudio: 640 bytes.
+    expect(decoded.sampleRateHz).toBe(ENGINE_RATE);
+    expect(decoded.audio.byteLength).toBe(640);
+
+    // The caller's final resample (reported rate → engine rate) is now an identity:
+    // 320 samples in, 320 samples out (not ~107 as under the old 48 kHz label → 3× fast).
+    const final = resampleAudioBytes(decoded.audio, decoded.sampleRateHz, ENGINE_RATE, new Map());
+    expect(final.byteLength).toBe(640);
+  });
+});

package/src/inbound-audio.ts

@@ -42,12 +42,19 @@ export function decodeInboundBinaryAudio(
   }
   const { header, audio } = decodeSyrinxAudioEnvelope(data);
   const sampleRateHz = requirePositiveIntegerFromHeader(header.sampleRateHz) ?? defaultSampleRateHz;
-  const wireAudio = header.encoding === "opus"
+  const isOpus = header.encoding === "opus";
+  const wireAudio = isOpus
     ? decodeOpusIngressMessage(audio, sampleRateHz, decodeOpusIngress, engineInputSampleRateHz, streamingResamplers)
     : audio;
+  // Opus ingress is decoded AND resampled to the engine rate inside
+  // decodeOpusIngressMessage, so the returned PCM is already at engineInputSampleRateHz.
+  // Report that as its rate — reporting the 48 kHz *header* rate made the caller
+  // resample the already-16 kHz audio a second time, delivering ~1/3 the samples
+  // (3× sped-up audio → STT garbage). PCM carries its true header source rate.
+  const effectiveSampleRateHz = isOpus ? engineInputSampleRateHz : sampleRateHz;
   return {
     contextId: typeof header.contextId === "string" && header.contextId.length > 0 ? header.contextId : undefined,
-    sampleRateHz,
+    sampleRateHz: effectiveSampleRateHz,
     sequence: header.sequence,
     audio: wireAudio,
   };

package/src/index.test.ts

@@ -601,7 +601,7 @@ describe("createVoiceWebSocketServer", () => {
       type: "tts_chunk",
       turnId: "turn-2",
       sequence: 1,
-      sampleRateHz: 16000,
+      sampleRateHz: 48000, // opus frames are labeled at the 48 kHz codec rate
       encoding: "opus",
       channels: 1,
     });
@@ -610,7 +610,7 @@ describe("createVoiceWebSocketServer", () => {
       type: "audio",
       contextId: "turn-2",
       sequence: 1,
-      sampleRateHz: 16000,
+      sampleRateHz: 48000,
       encoding: "opus",
     });
     expect(envelope.audio.byteLength).toBeGreaterThan(0);
@@ -1697,11 +1697,14 @@ describe("createVoiceWebSocketServer", () => {
     });
 
     const envelope = decodeTestBinaryAudioEnvelope(await binaryMessage);
+    // Opus frames are encoded at the codec rate (48 kHz) and MUST be labeled as such,
+    // so the client decodes then downsamples 48→16 kHz. Labeling them 16 kHz made the
+    // client skip the downsample and play assistant audio ~3× slow.
     expect(envelope.header).toMatchObject({
       type: "audio",
       contextId: "turn-tts",
       sequence: 1,
-      sampleRateHz: 16000,
+      sampleRateHz: 48000,
       encoding: "opus",
       channels: 1,
     });