@kuralle-syrinx/server-websocket 4.0.0 → 4.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/server-websocket",
-  "version": "4.0.0",
+  "version": "4.1.0",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -15,8 +15,8 @@
   "dependencies": {
     "@evan/opus": "1.0.3",
     "ws": "^8.18.0",
-    "@kuralle-syrinx/core": "4.0.0",
-    "@kuralle-syrinx/ws": "4.0.0"
+    "@kuralle-syrinx/core": "4.1.0",
+    "@kuralle-syrinx/ws": "4.1.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

package/src/background-audio.test.ts

@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+
+import { BackgroundAudioMixer } from "./background-audio.js";
+
+function pcmBytes(samples: number[]): Uint8Array {
+  return new Uint8Array(Int16Array.from(samples).buffer.slice(0));
+}
+
+function samplesOf(bytes: Uint8Array): number[] {
+  return Array.from(new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2));
+}
+
+/** Constant-value ambient source: every mixed sample adds value*gain. */
+function constantSource(value: number, length = 160, sampleRateHz = 16000) {
+  return { pcm: new Int16Array(length).fill(value), sampleRateHz };
+}
+
+describe("BackgroundAudioMixer", () => {
+  it("mixes ducked ambient into a TTS chunk and full-gain ambient into idle frames", () => {
+    const mixer = new BackgroundAudioMixer({
+      ambient: { ...constantSource(1000), gain: 0.5 },
+      duckWhileSpeaking: 0.5,
+      fadeMs: 0,
+    });
+
+    const t0 = 1_000_000;
+    const mixed = mixer.mix(pcmBytes([0, 0, 0, 0]), 16000, t0);
+    // ambient 1000 * gain 0.5 * duck 0.5 = 250 on top of silence
+    expect(samplesOf(mixed)).toEqual([250, 250, 250, 250]);
+
+    // Once speech has drained, the idle frame carries ambient at full gain.
+    const idle = mixer.idleFrame(1, 16000, t0 + 10_000);
+    expect(idle).not.toBeNull();
+    expect(samplesOf(idle!)).toEqual(Array(16).fill(500)); // 1ms @16k = 16 samples, 1000*0.5
+  });
+
+  it("suppresses idle frames while speech is still playing out", () => {
+    const mixer = new BackgroundAudioMixer({ ambient: constantSource(1000), fadeMs: 0 });
+    const t0 = 2_000_000;
+    // 160 samples @16k = 10ms of speech → speaking until t0+10.
+    mixer.mix(pcmBytes(Array(160).fill(0)), 16000, t0);
+    expect(mixer.isSpeaking(t0 + 5)).toBe(true);
+    expect(mixer.idleFrame(20, 16000, t0 + 5)).toBeNull();
+    expect(mixer.isSpeaking(t0 + 11)).toBe(false);
+    expect(mixer.idleFrame(20, 16000, t0 + 11)).not.toBeNull();
+  });
+
+  it("keeps the ambient position continuous across mix and idle frames", () => {
+    // Ramp source so position is observable: sample i has value i.
+    const ramp = Int16Array.from({ length: 1000 }, (_, i) => i);
+    const mixer = new BackgroundAudioMixer({
+      ambient: { pcm: ramp, sampleRateHz: 16000, gain: 1 },
+      duckWhileSpeaking: 1,
+      fadeMs: 0,
+    });
+    const t0 = 3_000_000;
+    const first = mixer.mix(pcmBytes([0, 0, 0, 0]), 16000, t0);
+    expect(samplesOf(first)).toEqual([0, 1, 2, 3]);
+    const idle = mixer.idleFrame(1, 16000, t0 + 1000);
+    expect(samplesOf(idle!)[0]).toBe(4); // continues where mix left off
+    const next = mixer.mix(pcmBytes([0, 0]), 16000, t0 + 2000);
+    expect(samplesOf(next)).toEqual([20, 21]); // 4 + 16 idle samples consumed
+  });
+
+  it("loops the ambient source past its end", () => {
+    const ramp = Int16Array.from({ length: 4 }, (_, i) => i + 1); // 1,2,3,4
+    const mixer = new BackgroundAudioMixer({
+      ambient: { pcm: ramp, sampleRateHz: 16000, gain: 1 },
+      duckWhileSpeaking: 1,
+      fadeMs: 0,
+    });
+    const mixed = mixer.mix(pcmBytes(Array(10).fill(0)), 16000, 4_000_000);
+    expect(samplesOf(mixed)).toEqual([1, 2, 3, 4, 1, 2, 3, 4, 1, 2]);
+  });
+
+  it("plays the thinking loop only while thinking, and restarts it fresh each episode", () => {
+    const ramp = Int16Array.from({ length: 100 }, (_, i) => i);
+    const mixer = new BackgroundAudioMixer({
+      thinking: { pcm: ramp, sampleRateHz: 16000, gain: 1 },
+      fadeMs: 0,
+    });
+    const t0 = 5_000_000;
+    expect(mixer.idleFrame(1, 16000, t0)).toBeNull(); // nothing to play
+
+    mixer.setThinking(true);
+    const a = mixer.idleFrame(1, 16000, t0);
+    expect(samplesOf(a!)).toEqual(Array.from({ length: 16 }, (_, i) => i));
+
+    mixer.setThinking(false);
+    expect(mixer.idleFrame(1, 16000, t0)).toBeNull();
+
+    mixer.setThinking(true); // new episode restarts at 0
+    const b = mixer.idleFrame(1, 16000, t0);
+    expect(samplesOf(b!)[0]).toBe(0);
+  });
+
+  it("resamples sources to the wire rate once and mixes correctly", () => {
+    // 8k constant source mixed into a 16k chunk: values unchanged, length follows the chunk.
+    const mixer = new BackgroundAudioMixer({
+      ambient: { pcm: new Int16Array(80).fill(800), sampleRateHz: 8000, gain: 1 },
+      duckWhileSpeaking: 1,
+      fadeMs: 0,
+    });
+    const mixed = mixer.mix(pcmBytes(Array(8).fill(0)), 16000, 6_000_000);
+    expect(samplesOf(mixed)).toEqual(Array(8).fill(800));
+  });
+
+  it("clips the sum to int16 range", () => {
+    const mixer = new BackgroundAudioMixer({
+      ambient: { pcm: new Int16Array(16).fill(20000), sampleRateHz: 16000, gain: 1 },
+      duckWhileSpeaking: 1,
+      fadeMs: 0,
+    });
+    const mixed = mixer.mix(pcmBytes([30000, -30000]), 16000, 7_000_000);
+    expect(samplesOf(mixed)).toEqual([32767, -10000]);
+  });
+
+  it("passes TTS through untouched when no sources are configured", () => {
+    const mixer = new BackgroundAudioMixer({});
+    const chunk = pcmBytes([1, 2, 3]);
+    expect(samplesOf(mixer.mix(chunk, 16000, 8_000_000))).toEqual([1, 2, 3]);
+    expect(mixer.idleFrame(20, 16000, 8_000_000)).toBeNull();
+    expect(mixer.hasSources).toBe(false);
+  });
+
+  it("fades the ambient bed in at start instead of hard-cutting (equal-power ramp)", () => {
+    // fadeMs 1 @ 16k = 16-sample ramp.
+    const mixer = new BackgroundAudioMixer({
+      ambient: { ...constantSource(1000), gain: 1 },
+      fadeMs: 1,
+    });
+    const first = mixer.idleFrame(1, 16000, 9_000_000)!;
+    const samples = samplesOf(first);
+    expect(samples[0]).toBe(0); // sin(0) = 0
+    expect(samples[8]!).toBeGreaterThan(400); // mid-ramp
+    expect(samples[8]!).toBeLessThan(900);
+    // Past the ramp: full gain, permanently.
+    const second = mixer.idleFrame(1, 16000, 9_000_100)!;
+    expect(samplesOf(second)).toEqual(Array(16).fill(1000));
+  });
+
+  it("fades the thinking loop out on stop, then goes silent", () => {
+    const mixer = new BackgroundAudioMixer({
+      thinking: { pcm: new Int16Array(1000).fill(1000), sampleRateHz: 16000, gain: 1 },
+      fadeMs: 1, // 16-sample ramps
+    });
+    const t0 = 10_000_000;
+    mixer.setThinking(true);
+    mixer.idleFrame(2, 16000, t0); // ride past the fade-in
+    const steady = mixer.idleFrame(1, 16000, t0 + 100)!;
+    expect(samplesOf(steady)).toEqual(Array(16).fill(1000));
+
+    mixer.setThinking(false);
+    const fading = mixer.idleFrame(1, 16000, t0 + 200)!; // one fade-out ramp, decaying
+    const fadeSamples = samplesOf(fading);
+    expect(fadeSamples[0]).toBe(1000); // cos(0) = 1
+    expect(fadeSamples[15]!).toBeLessThan(250); // near the end of the ramp
+    expect(mixer.idleFrame(1, 16000, t0 + 300)).toBeNull(); // episode fully over
+  });
+
+  it("fades each new thinking episode in from silence", () => {
+    const mixer = new BackgroundAudioMixer({
+      thinking: { pcm: new Int16Array(1000).fill(1000), sampleRateHz: 16000, gain: 1 },
+      fadeMs: 1,
+    });
+    mixer.setThinking(true);
+    const first = mixer.idleFrame(1, 16000, 11_000_000)!;
+    expect(samplesOf(first)[0]).toBe(0); // soft entry, not a hard cut
+  });
+});

package/src/background-audio.ts

@@ -0,0 +1,263 @@
+// SPDX-License-Identifier: MIT
+//
+// Background audio for the outbound path: a looped ambient bed (which doubles
+// as telephony comfort noise between turns), an optional "thinking" loop keyed
+// off the G3 tool-call cues, and ducking under assistant speech.
+//
+// Syrinx wires (Twilio media stream, browser jitter buffer) are single ORDERED
+// streams — unlike LiveKit's WebRTC rooms there is no client-side track mixing
+// — so background audio is mixed server-side into the one stream: `mix()` layers
+// the bed under every TTS chunk (ducked), and `idleFrame()` produces bed-only
+// frames for the gaps between turns. Source playback positions are shared
+// between the two so the bed is seamless across turn boundaries.
+//
+// Runtime-neutral: no Node imports — runs on workerd (edge hosts) as-is.
+// Prior art: Pipecat SoundfileMixer (transport-level mix + volume + loop),
+// LiveKit BackgroundAudioPlayer (ambient/thinking split, state-keyed thinking).
+
+import type { VoiceAgentSession } from "@kuralle-syrinx/core";
+import { pcm16BytesToSamples, pcm16SamplesToBytes, resamplePcm16 } from "@kuralle-syrinx/core/audio";
+
+export interface BackgroundAudioSource {
+  /** Mono PCM16 samples. Must be loop-clean if it will loop audibly. */
+  readonly pcm: Int16Array;
+  readonly sampleRateHz: number;
+  /** Linear gain 0..1 applied to this source. */
+  readonly gain?: number;
+}
+
+export interface BackgroundAudioConfig {
+  /** Looped continuously: under speech (ducked) and alone between turns. */
+  readonly ambient?: BackgroundAudioSource;
+  /** Looped while `setThinking(true)` — the audible face of a pending tool call. */
+  readonly thinking?: BackgroundAudioSource;
+  /**
+   * Extra multiplier on background sources while assistant speech is being
+   * mixed, so the bed never muddies the voice. 1 disables ducking. @default 0.5
+   */
+  readonly duckWhileSpeaking?: number;
+  /**
+   * Equal-power fade applied where the bed would otherwise hard-cut: the
+   * ambient's very first samples, and each thinking episode's start and stop.
+   * An abrupt full-volume onset is the harshest moment a caller hears from a
+   * wait — the fade is what makes an episode feel placed rather than switched.
+   * 0 disables. @default 250
+   */
+  readonly fadeMs?: number;
+}
+
+const DEFAULT_AMBIENT_GAIN = 0.25;
+const DEFAULT_THINKING_GAIN = 0.4;
+const DEFAULT_DUCK = 0.5;
+const DEFAULT_FADE_MS = 250;
+
+interface LoopedSource {
+  readonly original: BackgroundAudioSource;
+  readonly gain: number;
+  /** Resampled variants cached per wire rate. */
+  readonly byRate: Map<number, Int16Array>;
+  /** Playback position in samples at the current wire rate. */
+  position: number;
+  positionRateHz: number;
+}
+
+function loopedSource(source: BackgroundAudioSource, defaultGain: number): LoopedSource {
+  return {
+    original: source,
+    gain: source.gain ?? defaultGain,
+    byRate: new Map(),
+    position: 0,
+    positionRateHz: source.sampleRateHz,
+  };
+}
+
+function samplesAtRate(source: LoopedSource, rateHz: number): Int16Array {
+  let samples = source.byRate.get(rateHz);
+  if (!samples) {
+    samples = resamplePcm16(source.original.pcm, source.original.sampleRateHz, rateHz);
+    source.byRate.set(rateHz, samples);
+  }
+  if (source.positionRateHz !== rateHz) {
+    // Wire rate changed mid-connection (rare): carry the position over proportionally.
+    source.position = Math.round((source.position * rateHz) / source.positionRateHz);
+    source.positionRateHz = rateHz;
+  }
+  return samples;
+}
+
+/**
+ * Add samples of `source` (scaled by `scale`, optionally shaped per-sample by
+ * `envelope(i)`) into `target`, advancing the loop.
+ */
+function addLooped(
+  target: Float64Array,
+  source: LoopedSource,
+  rateHz: number,
+  scale: number,
+  envelope?: (i: number) => number,
+): void {
+  const samples = samplesAtRate(source, rateHz);
+  if (samples.length === 0 || scale === 0) return;
+  let pos = source.position % samples.length;
+  for (let i = 0; i < target.length; i += 1) {
+    const gain = envelope ? scale * envelope(i) : scale;
+    target[i]! += samples[pos]! * gain;
+    pos += 1;
+    if (pos >= samples.length) pos = 0;
+  }
+  source.position = pos;
+}
+
+/** Equal-power fade-in gain for sample `n` of a `total`-sample ramp (1 past the ramp). */
+function fadeInGain(n: number, total: number): number {
+  if (total <= 0 || n >= total) return 1;
+  return Math.sin((n / total) * (Math.PI / 2));
+}
+
+/** Equal-power fade-out gain for sample `n` of a `total`-sample ramp (0 past the ramp). */
+function fadeOutGain(n: number, total: number): number {
+  if (total <= 0 || n >= total) return 0;
+  return Math.cos((n / total) * (Math.PI / 2));
+}
+
+function clipToPcm16Bytes(mix: Float64Array): Uint8Array {
+  const out = new Int16Array(mix.length);
+  for (let i = 0; i < mix.length; i += 1) {
+    const v = Math.round(mix[i]!);
+    out[i] = v > 32767 ? 32767 : v < -32768 ? -32768 : v;
+  }
+  return pcm16SamplesToBytes(out);
+}
+
+export class BackgroundAudioMixer {
+  private readonly ambient: LoopedSource | null;
+  private readonly thinking: LoopedSource | null;
+  private readonly duck: number;
+  private readonly fadeMs: number;
+  /**
+   * Thinking episode lifecycle: "stopping" keeps the loop audible for one
+   * fade-out ramp after `setThinking(false)` instead of hard-cutting it.
+   */
+  private thinkingState: "off" | "on" | "stopping" = "off";
+  /** Samples rendered since the current thinking episode started (fade-in). */
+  private thinkingFadeInPos = 0;
+  /** Samples rendered since the stop was requested (fade-out). */
+  private thinkingFadeOutPos = 0;
+  /** Samples of ambient rendered so far (one-time fade-in at bed start). */
+  private ambientFadeInPos = 0;
+  /** Realtime estimate of when already-mixed speech finishes playing out. */
+  private speakingUntilMs = 0;
+
+  constructor(config: BackgroundAudioConfig) {
+    this.ambient = config.ambient ? loopedSource(config.ambient, DEFAULT_AMBIENT_GAIN) : null;
+    this.thinking = config.thinking ? loopedSource(config.thinking, DEFAULT_THINKING_GAIN) : null;
+    this.duck = config.duckWhileSpeaking ?? DEFAULT_DUCK;
+    this.fadeMs = config.fadeMs ?? DEFAULT_FADE_MS;
+  }
+
+  get hasSources(): boolean {
+    return this.ambient !== null || this.thinking !== null;
+  }
+
+  isSpeaking(nowMs = Date.now()): boolean {
+    return nowMs < this.speakingUntilMs;
+  }
+
+  /** G3 cue wiring: started/delayed → true, complete/failed → false. */
+  setThinking(on: boolean): void {
+    if (on) {
+      if (this.thinkingState === "on") return;
+      this.thinkingState = "on";
+      this.thinkingFadeInPos = 0;
+      if (this.thinking) this.thinking.position = 0; // each episode starts from the top
+      return;
+    }
+    if (this.thinkingState !== "on") return;
+    // Audible for one fade-out ramp, then off (immediately off when fades are disabled).
+    this.thinkingState = this.fadeMs > 0 ? "stopping" : "off";
+    this.thinkingFadeOutPos = 0;
+  }
+
+  /**
+   * Add the bed sources into `mixBuf` at `bedScale`, applying the fade
+   * envelopes and advancing their positions/counters. Shared by mix (ducked)
+   * and idleFrame (full gain) so episodes stay continuous across both.
+   */
+  private addBed(mixBuf: Float64Array, sampleRateHz: number, bedScale: number): void {
+    const fadeSamples = Math.round((this.fadeMs / 1000) * sampleRateHz);
+
+    if (this.ambient) {
+      const startPos = this.ambientFadeInPos;
+      const envelope = startPos >= fadeSamples
+        ? undefined
+        : (i: number) => fadeInGain(startPos + i, fadeSamples);
+      addLooped(mixBuf, this.ambient, sampleRateHz, this.ambient.gain * bedScale, envelope);
+      this.ambientFadeInPos = Math.min(fadeSamples, startPos + mixBuf.length);
+    }
+
+    if (this.thinking && this.thinkingState !== "off") {
+      if (this.thinkingState === "on") {
+        const startPos = this.thinkingFadeInPos;
+        const envelope = startPos >= fadeSamples
+          ? undefined
+          : (i: number) => fadeInGain(startPos + i, fadeSamples);
+        addLooped(mixBuf, this.thinking, sampleRateHz, this.thinking.gain * bedScale, envelope);
+        this.thinkingFadeInPos = Math.min(fadeSamples, startPos + mixBuf.length);
+      } else {
+        const startPos = this.thinkingFadeOutPos;
+        addLooped(
+          mixBuf,
+          this.thinking,
+          sampleRateHz,
+          this.thinking.gain * bedScale,
+          (i: number) => fadeOutGain(startPos + i, fadeSamples),
+        );
+        this.thinkingFadeOutPos = startPos + mixBuf.length;
+        if (this.thinkingFadeOutPos >= fadeSamples) this.thinkingState = "off";
+      }
+    }
+  }
+
+  /**
+   * Mix the background bed (ducked) into an assistant speech chunk and advance
+   * the speaking-until estimate by the chunk's realtime duration. Returns a new
+   * buffer; the input is never mutated.
+   */
+  mix(chunk: Uint8Array, sampleRateHz: number, nowMs = Date.now()): Uint8Array {
+    const speech = pcm16BytesToSamples(chunk);
+    const durationMs = (speech.length / sampleRateHz) * 1000;
+    this.speakingUntilMs = Math.max(this.speakingUntilMs, nowMs) + durationMs;
+
+    if (!this.hasSources) return chunk;
+    const mixBuf = new Float64Array(speech.length);
+    for (let i = 0; i < speech.length; i += 1) mixBuf[i] = speech[i]!;
+    this.addBed(mixBuf, sampleRateHz, this.duck);
+    return clipToPcm16Bytes(mixBuf);
+  }
+
+  /**
+   * Bed-only frame for the gaps between turns (telephony comfort noise), or
+   * null while speech is still playing out / there is nothing to play.
+   */
+  idleFrame(frameMs: number, sampleRateHz: number, nowMs = Date.now()): Uint8Array | null {
+    if (this.isSpeaking(nowMs)) return null;
+    const playThinking = this.thinking !== null && this.thinkingState !== "off";
+    if (!this.ambient && !playThinking) return null;
+
+    const sampleCount = Math.max(1, Math.round((frameMs / 1000) * sampleRateHz));
+    const mixBuf = new Float64Array(sampleCount);
+    this.addBed(mixBuf, sampleRateHz, 1);
+    return clipToPcm16Bytes(mixBuf);
+  }
+}
+
+/**
+ * Drive the mixer's thinking loop from the session's G3 tool-call cues:
+ * started/delayed → thinking on; complete/failed → off. Listener lifetime is
+ * the session's — both are per-connection.
+ */
+export function wireBackgroundThinking(session: VoiceAgentSession, mixer: BackgroundAudioMixer): void {
+  session.on("tool_call_cue", (event) => {
+    mixer.setThinking(event.phase === "started" || event.phase === "delayed");
+  });
+}

package/src/edge-twilio.test.ts

@@ -78,7 +78,10 @@ function bytesToBase64(bytes: Uint8Array): string {
   return Buffer.from(bytes).toString("base64");
 }
 
-async function startConnection(received: UserAudioReceivedPacket[] = []) {
+async function startConnection(
+  received: UserAudioReceivedPacket[] = [],
+  extraOptions: Record<string, unknown> = {},
+) {
   const socket = new FakeSocket();
   const session = fakeSession(received);
   await runTwilioEdgeWebSocketConnection(
@@ -88,6 +91,7 @@ async function startConnection(received: UserAudioReceivedPacket[] = []) {
       sessionStore: new InMemorySessionStore(),
       createSession: () => session,
       keepAliveIntervalMs: 0,
+      ...extraOptions,
     },
   );
   socket.emit(JSON.stringify({ event: "connected", protocol: "Call" }));
@@ -139,6 +143,54 @@ describe("Twilio edge ingress", () => {
     expect(Math.abs(decoded[100]! - 6000)).toBeLessThan(600);
   });
 
+  it("mixes the background bed (ducked) under TTS media", async () => {
+    const { socket, session } = await startConnection([], {
+      backgroundAudio: {
+        ambient: { pcm: new Int16Array(320).fill(4000), sampleRateHz: 16000, gain: 0.5 },
+        duckWhileSpeaking: 0.5,
+        fadeMs: 0,
+      },
+      backgroundIdleFrameMs: 10_000, // keep the idle ticker out of this test
+    });
+
+    const silence = new Uint8Array(new Int16Array(640).buffer.slice(0)); // 40ms @16k of silence
+    session.bus.push(Route.Critical, {
+      kind: "tts.audio",
+      contextId: "twilio-CA456",
+      timestampMs: Date.now(),
+      audio: silence,
+      sampleRateHz: 16000,
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    const media = socket.json().filter((msg) => msg.event === "media");
+    expect(media).toHaveLength(1);
+    const payload = (media[0]!.media as { payload: string }).payload;
+    const decoded = decodeMuLawToPcm16(new Uint8Array(Buffer.from(payload, "base64")));
+    // ambient 4000 × gain 0.5 × duck 0.5 = 1000 under the silent speech
+    expect(Math.abs(decoded[100]! - 1000)).toBeLessThan(120); // mu-law quantization tolerance
+  });
+
+  it("sends idle comfort-noise frames between turns", async () => {
+    const { socket } = await startConnection([], {
+      backgroundAudio: {
+        ambient: { pcm: new Int16Array(320).fill(4000), sampleRateHz: 8000, gain: 0.5 },
+        fadeMs: 0,
+      },
+      backgroundIdleFrameMs: 25,
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 120));
+
+    const media = socket.json().filter((msg) => msg.event === "media");
+    expect(media.length).toBeGreaterThanOrEqual(2); // ticker fills the silence
+    const payload = (media[0]!.media as { payload: string }).payload;
+    const decoded = decodeMuLawToPcm16(new Uint8Array(Buffer.from(payload, "base64")));
+    // full-gain bed between turns: 4000 × 0.5 = 2000
+    expect(Math.abs(decoded[10]! - 2000)).toBeLessThan(220);
+    expect(decoded.length).toBe(200); // 25ms @ 8k
+  });
+
   it("sends a clear event on interrupt.detected (barge-in)", async () => {
     const { socket, session } = await startConnection();
 

package/src/edge-twilio.ts

@@ -26,6 +26,11 @@ import {
   type StreamingPcm16Resampler,
 } from "@kuralle-syrinx/core/audio";
 import type { ManagedSocket, SocketData } from "@kuralle-syrinx/ws";
+import {
+  BackgroundAudioMixer,
+  wireBackgroundThinking,
+  type BackgroundAudioConfig,
+} from "./background-audio.js";
 import {
   createWorkersInboundSocket,
   type WorkersDurableObjectWebSocketContext,
@@ -45,6 +50,14 @@ export interface TwilioEdgeWebSocketOptions {
   readonly scheduler?: Scheduler;
   /** Engine-side PCM rate (default 16000). Twilio's leg is always 8 kHz μ-law. */
   readonly engineSampleRateHz?: number;
+  /**
+   * Ambient/thinking bed: mixed (ducked) under assistant speech, and sent as
+   * comfort-noise frames between turns — pure digital silence on a phone line
+   * reads as "the call died". Thinking loop follows the G3 tool-call cues.
+   */
+  readonly backgroundAudio?: BackgroundAudioConfig;
+  /** Cadence of idle comfort-noise frames (ms). @default 200 */
+  readonly backgroundIdleFrameMs?: number;
   readonly resumeWindowMs?: number;
   readonly keepAliveIntervalMs?: number;
   readonly idleTimeoutMs?: number;
@@ -187,13 +200,34 @@ export async function runTwilioEdgeWebSocketConnection(
       return;
     }
 
+    // Background bed: ducked under speech in the tts.audio handler below, and
+    // sent as idle comfort-noise frames between turns — a phone line carrying
+    // pure digital silence reads as "the call died". Twilio's `clear` on
+    // barge-in drops any queued bed audio; the ticker refills on the next tick.
+    const backgroundAudio = options.backgroundAudio
+      ? new BackgroundAudioMixer(options.backgroundAudio)
+      : null;
+    if (backgroundAudio) {
+      wireBackgroundThinking(session, backgroundAudio);
+      const idleFrameMs = options.backgroundIdleFrameMs ?? 200;
+      const idleTimer = setInterval(() => {
+        if (!streamSid || stopped || closed || !socket.isOpen) return;
+        const frame = backgroundAudio.idleFrame(idleFrameMs, TWILIO_SAMPLE_RATE_HZ);
+        if (!frame) return;
+        const mulaw = encodePcm16ToMuLaw(pcm16BytesToSamples(frame));
+        sendTwilioJson({ event: "media", streamSid, media: { payload: bytesToBase64(mulaw) } });
+      }, idleFrameMs);
+      disposers.push(() => clearInterval(idleTimer));
+    }
+
     // Downlink: engine PCM → 8 kHz μ-law media frames; barge-in → clear.
     disposers.push(
       session.bus.on("tts.audio", (pkt) => {
         if (!streamSid) return;
         const audio = pkt as TextToSpeechAudioPacket;
-        const samples = pcm16BytesToSamples(audio.audio);
         const sourceRate = audio.sampleRateHz ?? engineRate;
+        const wireAudio = backgroundAudio ? backgroundAudio.mix(audio.audio, sourceRate) : audio.audio;
+        const samples = pcm16BytesToSamples(wireAudio);
         const resampled = resamplePcm16Streaming(downlinkResamplers, samples, sourceRate, TWILIO_SAMPLE_RATE_HZ);
         const mulaw = encodePcm16ToMuLaw(resampled);
         sendTwilioJson({

package/src/edge.test.ts

@@ -529,3 +529,63 @@ describe("edge client playout progress", () => {
     expect(socket.json().some((m) => m.type === "error" && m.category === "invalid_input")).toBe(true);
   });
 });
+
+describe("edge background audio", () => {
+  it("mixes the bed into the wire audio while the recorder keeps the clean track", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    const recorded: Int16Array[] = [];
+    const rec: EdgeRecorder = {
+      onUserAudio() {},
+      onAssistantAudio(_contextId: string, audio: Uint8Array) {
+        recorded.push(new Int16Array(audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength)));
+      },
+      finalize() {},
+    };
+    const bus = new PipelineBusImpl();
+    const session = {
+      bus,
+      async start() {
+        void bus.start();
+      },
+      async close() {
+        bus.stop();
+      },
+      on() {},
+      off() {},
+      requestClientInterrupt() {},
+    } as unknown as VoiceAgentSession;
+
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => session,
+      recorder: rec,
+      backgroundAudio: {
+        ambient: { pcm: new Int16Array(160).fill(2000), sampleRateHz: 16000, gain: 0.5 },
+        duckWhileSpeaking: 0.5,
+        fadeMs: 0,
+      },
+    });
+    waitForReady(socket);
+
+    bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "bg-turn",
+      timestampMs: Date.now(),
+      audio: pcm16SamplesToBytes(new Int16Array(4)), // silent speech chunk
+      sampleRateHz: 16000,
+    });
+    await new Promise((r) => setTimeout(r, 20));
+
+    // Recorder saw the CLEAN chunk.
+    expect(recorded).toHaveLength(1);
+    expect(Array.from(recorded[0]!)).toEqual([0, 0, 0, 0]);
+
+    // The wire envelope carries the bed: 2000 × 0.5 gain × 0.5 duck = 500.
+    const binary = socket.sent.find((d): d is Uint8Array => d instanceof Uint8Array);
+    expect(binary).toBeDefined();
+    const wireSamples = new Int16Array(binary!.buffer.slice(binary!.byteOffset + (binary!.byteLength - 8)));
+    expect(Array.from(wireSamples)).toEqual([500, 500, 500, 500]);
+  });
+});

package/src/edge.ts

@@ -14,6 +14,12 @@ import {
 } from "@kuralle-syrinx/core";
 import { TimerScheduler, type Scheduler } from "@kuralle-syrinx/core";
 import { type StreamingPcm16Resampler } from "@kuralle-syrinx/core/audio";
+import {
+  BackgroundAudioMixer,
+  wireBackgroundThinking,
+  type BackgroundAudioConfig,
+} from "./background-audio.js";
+export type { BackgroundAudioConfig, BackgroundAudioSource } from "./background-audio.js";
 import {
   createWorkersInboundSocket,
   type WorkersDurableObjectWebSocketContext,
@@ -76,6 +82,12 @@ export interface VoiceEdgeWebSocketOptions {
    * the Syrinx binary envelope instead.
    */
   readonly rawBinaryInput?: boolean;
+  /**
+   * Ambient/thinking bed mixed (ducked) under assistant speech. Browser edge
+   * sends no idle bed between turns — a web client can loop ambience locally;
+   * the server-side bed exists for wires with no client runtime (telephony).
+   */
+  readonly backgroundAudio?: BackgroundAudioConfig;
   readonly sessionStore: SessionStore;
   readonly scheduler?: Scheduler;
 }
@@ -265,7 +277,14 @@ export async function runVoiceEdgeWebSocketConnection(
       await options.sessionStore.release(leased.managed.id, 0);
       return;
     }
-    wireEdgeSessionEvents(session, socket, disposers, outputSampleRateHz, options.recorder);
+    wireEdgeSessionEvents(
+      session,
+      socket,
+      disposers,
+      outputSampleRateHz,
+      options.recorder,
+      options.backgroundAudio ? new BackgroundAudioMixer(options.backgroundAudio) : undefined,
+    );
     if (options.recorder) {
       const recorder = options.recorder;
       disposers.push(
@@ -349,7 +368,9 @@ function wireEdgeSessionEvents(
   disposers: Array<() => void>,
   outputSampleRateHz: number,
   recorder?: EdgeRecorder,
+  backgroundAudio?: BackgroundAudioMixer,
 ): void {
+  if (backgroundAudio) wireBackgroundThinking(session, backgroundAudio);
   const onSession = <K extends keyof VoiceAgentSessionEvents>(
     event: K,
     handler: VoiceAgentSessionEvents[K],
@@ -391,27 +412,29 @@ function wireEdgeSessionEvents(
     session.bus.on("tts.audio", (pkt) => {
       const audio = pkt as TextToSpeechAudioPacket;
       const ttsSampleRate = requireTtsAudioSampleRate(audio.sampleRateHz) ?? outputSampleRateHz;
+      // Recorder gets the CLEAN assistant track; the bed is a wire-level effect.
       recorder?.onAssistantAudio(audio.contextId, audio.audio, ttsSampleRate);
+      const wireAudio = backgroundAudio ? backgroundAudio.mix(audio.audio, ttsSampleRate) : audio.audio;
       sendJson(socket, {
         type: "tts_chunk",
         turnId: audio.contextId,
         sequence: 1,
-        sampleRateHz: requireTtsAudioSampleRate(audio.sampleRateHz) ?? outputSampleRateHz,
+        sampleRateHz: ttsSampleRate,
         encoding: "pcm_s16le",
         channels: 1,
-        byteLength: audio.audio.byteLength,
-        durationMs: pcm16DurationMs(audio.audio, outputSampleRateHz),
+        byteLength: wireAudio.byteLength,
+        durationMs: pcm16DurationMs(wireAudio, outputSampleRateHz),
       });
       socket.send(encodeSyrinxAudioEnvelope({
         type: "audio",
         contextId: audio.contextId,
         sequence: 1,
-        sampleRateHz: requireTtsAudioSampleRate(audio.sampleRateHz) ?? outputSampleRateHz,
+        sampleRateHz: ttsSampleRate,
         encoding: "pcm_s16le",
         channels: 1,
-        byteLength: audio.audio.byteLength,
-        durationMs: pcm16DurationMs(audio.audio, outputSampleRateHz),
-      }, audio.audio));
+        byteLength: wireAudio.byteLength,
+        durationMs: pcm16DurationMs(wireAudio, outputSampleRateHz),
+      }, wireAudio));
     }),
     session.bus.on("tts.end", (pkt) => {
       const end = pkt as TextToSpeechEndPacket;

package/src/index.ts

@@ -32,6 +32,7 @@ import { closeWebSocketWithFallback, waitForWebSocketClose } from "./websocket-c
 import { isRecord, parseJsonRecord, optionalString, requiredString } from "./json-message.js";
 import { createRoutedWebSocketServer } from "./websocket-upgrade.js";
 import { runWebSocketConnection, type GracefulCloseOptions, type TransportAdapter, type TransportHostConfig, TRANSPORT_ADMISSION_REJECTED_METRIC } from "./transport-host.js";
+import { BackgroundAudioMixer, type BackgroundAudioConfig } from "./background-audio.js";
 import { wireTelephonyOutboundPipeline, type TelephonyOutboundCallbacks, type TelephonyOutboundHandle } from "./outbound-playout-pipeline.js";
 import { TurnMetricsTracker, type TurnTimestampState } from "./turn-metrics.js";
 import { type PacedPlayoutFrame } from "./paced-playout.js";
@@ -59,6 +60,12 @@ export * from "./telnyx.js";
 export * from "./smartpbx.js";
 export * from "./session-store.js";
 export { validateTwilioSignature } from "./twilio-auth.js";
+export {
+  BackgroundAudioMixer,
+  wireBackgroundThinking,
+  type BackgroundAudioConfig,
+  type BackgroundAudioSource,
+} from "./background-audio.js";
 export { installGracefulShutdown, type GracefulClosable } from "./websocket-lifecycle.js";
 
 export interface VoiceWebSocketServerOptions {
@@ -97,6 +104,12 @@ export interface VoiceWebSocketServerOptions {
    * envelope. Set false only for tests or legacy clients that require PCM envelopes.
    */
   readonly browserOpusDownlink?: boolean;
+  /**
+   * Ambient/thinking bed mixed (ducked) under assistant speech. No idle bed is
+   * sent between turns on the browser path — a web client can loop ambience
+   * locally; the server-side idle bed exists for telephony wires.
+   */
+  readonly backgroundAudio?: BackgroundAudioConfig;
   readonly sessionStore?: SessionStore;
   readonly maxConcurrentSessions?: number;
   readonly maxConcurrentSessionsScope?: "path" | "server";
@@ -253,6 +266,7 @@ export async function createVoiceWebSocketServer(
         () => state.browserOpusDownlink,
         managed.turnMetricsTurns,
         state.streamingResamplers,
+        options.backgroundAudio ? new BackgroundAudioMixer(options.backgroundAudio) : undefined,
       );
       gracefulCloseRegistry.set(socket, (deadlineMs) => {
         if (socket.readyState === WebSocket.CLOSED || socket.readyState === WebSocket.CLOSING) {
@@ -432,6 +446,7 @@ function wireBrowserSessionEvents(
   getBrowserOpusDownlink: () => boolean,
   turnMetricsTurns: Map<string, TurnTimestampState>,
   streamingResamplers: Map<string, StreamingPcm16Resampler>,
+  backgroundAudio?: BackgroundAudioMixer,
 ): TelephonyOutboundHandle {
   const ttsSequences = new Map<string, number>();
   const speechEndedSent = new Set<string>();
@@ -656,6 +671,7 @@ function wireBrowserSessionEvents(
     outboundFrameDurationMs,
     maxQueuedOutputAudioMs,
     callbacks,
+    ...(backgroundAudio ? { backgroundAudio } : {}),
   });
 }
 

package/src/outbound-playout-pipeline.test.ts

@@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest";
 import WebSocket from "ws";
 import { Route, VoiceAgentSession } from "@kuralle-syrinx/core";
 import { pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
+import { BackgroundAudioMixer } from "./background-audio.js";
 import { wireTelephonyOutboundPipeline, installTelephonyTurnRotation, type RotatableTurnContext } from "./outbound-playout-pipeline.js";
 
 function createMockSocket(): WebSocket {
@@ -262,4 +263,52 @@ describe("installTelephonyTurnRotation", () => {
     for (const dispose of disposers) dispose();
     await session.close();
   });
+
+  it("mixes the background bed into speech before per-carrier encoding", async () => {
+    const socket = createMockSocket();
+    const session = new VoiceAgentSession({ plugins: {} });
+    void session.start();
+    const disposers: Array<() => void> = [];
+    const encodedChunks: Int16Array[] = [];
+    wireTelephonyOutboundPipeline({
+      session,
+      socket,
+      disposers,
+      outboundFrameDurationMs: 20,
+      maxQueuedOutputAudioMs: 30_000,
+      backgroundAudio: new BackgroundAudioMixer({
+        ambient: { pcm: new Int16Array(160).fill(1000), sampleRateHz: 16000, gain: 0.5 },
+        duckWhileSpeaking: 0.5,
+        fadeMs: 0,
+      }),
+      callbacks: {
+        carrierLabel: "test",
+        getContextId: () => "turn-mix",
+        isActive: () => true,
+        encodeFrames: (audio) => {
+          encodedChunks.push(new Int16Array(audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength)));
+          return [{ contextId: "turn-mix", send: () => true }];
+        },
+        onInterrupt: () => undefined,
+        onDrain: () => undefined,
+        onStop: () => undefined,
+      },
+    });
+
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-mix",
+      timestampMs: Date.now(),
+      audio: pcm16SamplesToBytes(new Int16Array(4)), // silence in → ambient-only out
+      sampleRateHz: 16000,
+    });
+    await new Promise((r) => setTimeout(r, 20));
+
+    expect(encodedChunks).toHaveLength(1);
+    // ambient 1000 × gain 0.5 × duck 0.5 = 250 layered under the (silent) speech
+    expect(Array.from(encodedChunks[0]!)).toEqual([250, 250, 250, 250]);
+
+    for (const dispose of disposers) dispose();
+    await session.close();
+  });
 });

package/src/outbound-playout-pipeline.ts

@@ -2,6 +2,8 @@
 
 import { Route, type InterruptTtsPacket, type TextToSpeechAudioPacket, type TextToSpeechEndPacket, type VoiceAgentSession } from "@kuralle-syrinx/core";
 import { WebSocket } from "ws";
+import type { BackgroundAudioMixer } from "./background-audio.js";
+import { wireBackgroundThinking } from "./background-audio.js";
 import type { PacedPlayoutFrame } from "./paced-playout.js";
 import { PacedPlayoutQueue } from "./paced-playout.js";
 import { PlayoutProgressEmitter } from "./playout-progress.js";
@@ -70,8 +72,16 @@ export function wireTelephonyOutboundPipeline(args: {
   readonly outboundFrameDurationMs: number;
   readonly maxQueuedOutputAudioMs: number;
   readonly callbacks: TelephonyOutboundCallbacks;
+  /**
+   * Mixes the ambient/thinking bed under every outbound speech chunk (ducked).
+   * Idle comfort-noise frames are not generated on this path yet — the paced
+   * queue's per-encode mark machinery needs a per-carrier background-frame
+   * encoder first (see background-audio-implementation-notes.md).
+   */
+  readonly backgroundAudio?: BackgroundAudioMixer;
 }): TelephonyOutboundHandle {
-  const { session, socket, disposers, outboundFrameDurationMs, maxQueuedOutputAudioMs, callbacks } = args;
+  const { session, socket, disposers, outboundFrameDurationMs, maxQueuedOutputAudioMs, callbacks, backgroundAudio } = args;
+  if (backgroundAudio) wireBackgroundThinking(session, backgroundAudio);
 
   const recordDiscardedPlayout = (discardedMs: number, reason: string): void => {
     if (discardedMs <= 0) return;
@@ -147,11 +157,11 @@ export function wireTelephonyOutboundPipeline(args: {
         });
         return;
       }
-      const frames = callbacks.encodeFrames(
-        audioPacket.audio,
-        requireTtsAudioSampleRate(audioPacket.sampleRateHz),
-        audioPacket.contextId,
-      );
+      const sampleRateHz = requireTtsAudioSampleRate(audioPacket.sampleRateHz);
+      const audio = backgroundAudio
+        ? backgroundAudio.mix(audioPacket.audio, sampleRateHz)
+        : audioPacket.audio;
+      const frames = callbacks.encodeFrames(audio, sampleRateHz, audioPacket.contextId);
       playout.enqueue(frames);
     }),
     session.bus.on("tts.end", (pkt) => {

package/src/smartpbx.ts

@@ -23,6 +23,7 @@ import {
 } from "./json-message.js";
 import { createRoutedWebSocketServer } from "./websocket-upgrade.js";
 import { runWebSocketConnection, type GracefulCloseOptions, type TransportAdapter, type TransportHostConfig, TRANSPORT_ADMISSION_REJECTED_METRIC } from "./transport-host.js";
+import { BackgroundAudioMixer, type BackgroundAudioConfig } from "./background-audio.js";
 import { wireTelephonyOutboundPipeline, installTelephonyTurnRotation } from "./outbound-playout-pipeline.js";
 import {
   decodeStrictBase64,
@@ -43,6 +44,8 @@ export interface SmartPbxMediaStreamServerOptions {
   readonly outputSampleRateHz?: number;
   readonly outboundFrameDurationMs?: number;
   readonly maxQueuedOutputAudioMs?: number;
+  /** Ambient/thinking bed mixed (ducked) under assistant speech (see BackgroundAudioConfig). */
+  readonly backgroundAudio?: BackgroundAudioConfig;
   readonly heartbeatIntervalMs?: number;
   readonly startupTimeoutMs?: number;
   readonly maxSessionDurationMs?: number;
@@ -179,6 +182,7 @@ export async function createSmartPbxMediaStreamServer(
         disposers,
         outboundFrameDurationMs,
         maxQueuedOutputAudioMs,
+        ...(options.backgroundAudio ? { backgroundAudio: new BackgroundAudioMixer(options.backgroundAudio) } : {}),
         callbacks: {
           carrierLabel: "smartpbx",
           getContextId: () => state.contextId,

package/src/telnyx.ts

@@ -25,6 +25,7 @@ import {
 } from "./json-message.js";
 import { createRoutedWebSocketServer } from "./websocket-upgrade.js";
 import { runWebSocketConnection, type GracefulCloseOptions, type TransportAdapter, type TransportHostConfig, TRANSPORT_ADMISSION_REJECTED_METRIC } from "./transport-host.js";
+import { BackgroundAudioMixer, type BackgroundAudioConfig } from "./background-audio.js";
 import { wireTelephonyOutboundPipeline, installTelephonyTurnRotation } from "./outbound-playout-pipeline.js";
 import {
   decodeStrictBase64,
@@ -49,6 +50,8 @@ export interface TelnyxMediaStreamServerOptions {
   readonly bidirectionalCodec?: "PCMU" | "L16";
   readonly outboundFrameDurationMs?: number;
   readonly maxQueuedOutputAudioMs?: number;
+  /** Ambient/thinking bed mixed (ducked) under assistant speech (see BackgroundAudioConfig). */
+  readonly backgroundAudio?: BackgroundAudioConfig;
   readonly maxInboundReorderFrames?: number;
   readonly heartbeatIntervalMs?: number;
   readonly startupTimeoutMs?: number;
@@ -224,6 +227,7 @@ export async function createTelnyxMediaStreamServer(
         disposers,
         outboundFrameDurationMs,
         maxQueuedOutputAudioMs,
+        ...(options.backgroundAudio ? { backgroundAudio: new BackgroundAudioMixer(options.backgroundAudio) } : {}),
         callbacks: {
           carrierLabel: "telnyx",
           getContextId: () => state.contextId,

package/src/twilio.ts

@@ -22,6 +22,7 @@ import {
 } from "./json-message.js";
 import { createRoutedWebSocketServer } from "./websocket-upgrade.js";
 import { runWebSocketConnection, type GracefulCloseOptions, type TransportAdapter, type TransportHostConfig, TRANSPORT_ADMISSION_REJECTED_METRIC } from "./transport-host.js";
+import { BackgroundAudioMixer, type BackgroundAudioConfig } from "./background-audio.js";
 import { wireTelephonyOutboundPipeline, installTelephonyTurnRotation } from "./outbound-playout-pipeline.js";
 import {
   decodeStrictBase64,
@@ -45,6 +46,8 @@ export interface TwilioMediaStreamServerOptions {
   readonly twilioSampleRateHz?: number;
   readonly outboundFrameDurationMs?: number;
   readonly maxQueuedOutputAudioMs?: number;
+  /** Ambient/thinking bed mixed (ducked) under assistant speech (see BackgroundAudioConfig). */
+  readonly backgroundAudio?: BackgroundAudioConfig;
   readonly heartbeatIntervalMs?: number;
   readonly startupTimeoutMs?: number;
   readonly maxSessionDurationMs?: number;
@@ -200,6 +203,7 @@ export async function createTwilioMediaStreamServer(
         disposers,
         outboundFrameDurationMs,
         maxQueuedOutputAudioMs,
+        ...(options.backgroundAudio ? { backgroundAudio: new BackgroundAudioMixer(options.backgroundAudio) } : {}),
         callbacks: {
           carrierLabel: "twilio",
           getContextId: () => state.contextId,