@kuralle-syrinx/server-websocket 4.1.0 → 4.3.0

package/package.json

@@ -1,28 +1,48 @@
 {
   "name": "@kuralle-syrinx/server-websocket",
-  "version": "4.1.0",
+  "version": "4.3.0",
   "private": false,
-  "type": "module",
+  "description": "Node WebSocket voice host for Syrinx — browser transport plus Twilio/Telnyx/SmartPBX telephony adapters, background audio, admission control",
+  "keywords": [
+    "voice",
+    "voice-agent",
+    "speech",
+    "syrinx",
+    "telephony",
+    "twilio",
+    "telnyx"
+  ],
   "license": "MIT",
+  "homepage": "https://github.com/kuralle/syrinx#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kuralle/syrinx.git",
+    "directory": "packages/server-websocket"
+  },
+  "bugs": {
+    "url": "https://github.com/kuralle/syrinx/issues"
+  },
+  "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
   "exports": {
     ".": "./src/index.ts",
     "./edge": "./src/edge.ts",
     "./edge-twilio": "./src/edge-twilio.ts",
+    "./edge-telnyx": "./src/edge-telnyx.ts",
     "./session-store": "./src/session-store.ts"
   },
   "dependencies": {
     "@evan/opus": "1.0.3",
-    "ws": "^8.18.0",
-    "@kuralle-syrinx/core": "4.1.0",
-    "@kuralle-syrinx/ws": "4.1.0"
+    "ws": "^8.21.0",
+    "@kuralle-syrinx/core": "4.3.0",
+    "@kuralle-syrinx/ws": "4.3.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",
     "@types/ws": "^8.5.0",
     "typescript": "^5.7.0",
-    "vitest": "^2.1.0"
+    "vitest": "^3.2.6"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",

package/src/backchannel-cue-fixtures.ts

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: MIT
+//
+// Deterministic placeholder backchannel cue PCM (16 kHz mono). Real voice-matched
+// assets + locale table are a follow-up — these shaped tones prove the byte-config path.
+
+import { synthesizeTonePcm16 } from "@kuralle-syrinx/core";
+import { pcm16BytesToSamples } from "@kuralle-syrinx/core/audio";
+import type { BackgroundAudioSource } from "./background-audio.js";
+
+const CUE_SAMPLE_RATE_HZ = 16_000;
+
+function placeholderCue(frequencyHz: number, durationMs: number, gain: number): BackgroundAudioSource {
+  const bytes = synthesizeTonePcm16({
+    frequencyHz,
+    durationMs,
+    sampleRateHz: CUE_SAMPLE_RATE_HZ,
+    amplitude: 0.35,
+  });
+  return { pcm: pcm16BytesToSamples(bytes), sampleRateHz: CUE_SAMPLE_RATE_HZ, gain };
+}
+
+/** Placeholder cue map — pass as `backgroundAudio.cues` at session init. */
+export function buildPlaceholderBackchannelCues(): Readonly<Record<string, BackgroundAudioSource>> {
+  return {
+    mm_hmm: placeholderCue(440, 400, 0.65),
+    yeah: placeholderCue(520, 350, 0.6),
+    got_it: placeholderCue(380, 450, 0.6),
+  };
+}

package/src/background-audio.ts

@@ -2,7 +2,7 @@
 //
 // Background audio for the outbound path: a looped ambient bed (which doubles
 // as telephony comfort noise between turns), an optional "thinking" loop keyed
-// off the G3 tool-call cues, and ducking under assistant speech.
+// off the G3 tool-call cues, one-shot backchannel cues, and ducking under speech.
 //
 // Syrinx wires (Twilio media stream, browser jitter buffer) are single ORDERED
 // streams — unlike LiveKit's WebRTC rooms there is no client-side track mixing
@@ -15,7 +15,8 @@
 // Prior art: Pipecat SoundfileMixer (transport-level mix + volume + loop),
 // LiveKit BackgroundAudioPlayer (ambient/thinking split, state-keyed thinking).
 
-import type { VoiceAgentSession } from "@kuralle-syrinx/core";
+import type { InteractionBackchannelPacket, VoiceAgentSession } from "@kuralle-syrinx/core";
+import { Route } from "@kuralle-syrinx/core";
 import { pcm16BytesToSamples, pcm16SamplesToBytes, resamplePcm16 } from "@kuralle-syrinx/core/audio";
 
 export interface BackgroundAudioSource {
@@ -31,6 +32,8 @@ export interface BackgroundAudioConfig {
   readonly ambient?: BackgroundAudioSource;
   /** Looped while `setThinking(true)` — the audible face of a pending tool call. */
   readonly thinking?: BackgroundAudioSource;
+  /** Pre-cached one-shot backchannel cues keyed by cue id (IP-C3). */
+  readonly cues?: Readonly<Record<string, BackgroundAudioSource>>;
   /**
    * Extra multiplier on background sources while assistant speech is being
    * mixed, so the bed never muddies the voice. 1 disables ducking. @default 0.5
@@ -48,8 +51,10 @@ export interface BackgroundAudioConfig {
 
 const DEFAULT_AMBIENT_GAIN = 0.25;
 const DEFAULT_THINKING_GAIN = 0.4;
+const DEFAULT_CUE_GAIN = 0.65;
 const DEFAULT_DUCK = 0.5;
 const DEFAULT_FADE_MS = 250;
+const CUE_THINKING_DUCK = 0.25;
 
 interface LoopedSource {
   readonly original: BackgroundAudioSource;
@@ -61,6 +66,14 @@ interface LoopedSource {
   positionRateHz: number;
 }
 
+interface OneShotSource {
+  readonly original: BackgroundAudioSource;
+  readonly gain: number;
+  readonly byRate: Map<number, Int16Array>;
+  position: number;
+  positionRateHz: number;
+}
+
 function loopedSource(source: BackgroundAudioSource, defaultGain: number): LoopedSource {
   return {
     original: source,
@@ -71,14 +84,26 @@ function loopedSource(source: BackgroundAudioSource, defaultGain: number): Loope
   };
 }
 
-function samplesAtRate(source: LoopedSource, rateHz: number): Int16Array {
+function oneShotSource(source: BackgroundAudioSource, defaultGain: number): OneShotSource {
+  return {
+    original: source,
+    gain: source.gain ?? defaultGain,
+    byRate: new Map(),
+    position: 0,
+    positionRateHz: source.sampleRateHz,
+  };
+}
+
+function samplesAtRate(
+  source: LoopedSource | OneShotSource,
+  rateHz: number,
+): Int16Array {
   let samples = source.byRate.get(rateHz);
   if (!samples) {
     samples = resamplePcm16(source.original.pcm, source.original.sampleRateHz, rateHz);
     source.byRate.set(rateHz, samples);
   }
   if (source.positionRateHz !== rateHz) {
-    // Wire rate changed mid-connection (rare): carry the position over proportionally.
     source.position = Math.round((source.position * rateHz) / source.positionRateHz);
     source.positionRateHz = rateHz;
   }
@@ -108,6 +133,22 @@ function addLooped(
   source.position = pos;
 }
 
+function addOneShot(target: Float64Array, source: OneShotSource, rateHz: number): boolean {
+  const samples = samplesAtRate(source, rateHz);
+  if (samples.length === 0 || source.gain === 0) return true;
+  let pos = source.position;
+  for (let i = 0; i < target.length; i += 1) {
+    if (pos >= samples.length) {
+      source.position = samples.length;
+      return true;
+    }
+    target[i]! += samples[pos]! * source.gain;
+    pos += 1;
+  }
+  source.position = pos;
+  return pos >= samples.length;
+}
+
 /** Equal-power fade-in gain for sample `n` of a `total`-sample ramp (1 past the ramp). */
 function fadeInGain(n: number, total: number): number {
   if (total <= 0 || n >= total) return 1;
@@ -132,59 +173,65 @@ function clipToPcm16Bytes(mix: Float64Array): Uint8Array {
 export class BackgroundAudioMixer {
   private readonly ambient: LoopedSource | null;
   private readonly thinking: LoopedSource | null;
+  private readonly cueCatalog: ReadonlyMap<string, BackgroundAudioSource>;
   private readonly duck: number;
   private readonly fadeMs: number;
-  /**
-   * Thinking episode lifecycle: "stopping" keeps the loop audible for one
-   * fade-out ramp after `setThinking(false)` instead of hard-cutting it.
-   */
   private thinkingState: "off" | "on" | "stopping" = "off";
-  /** Samples rendered since the current thinking episode started (fade-in). */
   private thinkingFadeInPos = 0;
-  /** Samples rendered since the stop was requested (fade-out). */
   private thinkingFadeOutPos = 0;
-  /** Samples of ambient rendered so far (one-time fade-in at bed start). */
   private ambientFadeInPos = 0;
-  /** Realtime estimate of when already-mixed speech finishes playing out. */
   private speakingUntilMs = 0;
+  private pendingCue: OneShotSource | null = null;
 
   constructor(config: BackgroundAudioConfig) {
     this.ambient = config.ambient ? loopedSource(config.ambient, DEFAULT_AMBIENT_GAIN) : null;
     this.thinking = config.thinking ? loopedSource(config.thinking, DEFAULT_THINKING_GAIN) : null;
+    this.cueCatalog = new Map(Object.entries(config.cues ?? {}));
     this.duck = config.duckWhileSpeaking ?? DEFAULT_DUCK;
     this.fadeMs = config.fadeMs ?? DEFAULT_FADE_MS;
   }
 
   get hasSources(): boolean {
-    return this.ambient !== null || this.thinking !== null;
+    return this.ambient !== null || this.thinking !== null || this.cueCatalog.size > 0;
+  }
+
+  hasCue(cueId: string): boolean {
+    return this.cueCatalog.has(cueId);
   }
 
   isSpeaking(nowMs = Date.now()): boolean {
     return nowMs < this.speakingUntilMs;
   }
 
+  /** Queue a one-shot backchannel cue by id. Returns false when the cue is unknown. */
+  queueCue(cueId: string): boolean {
+    const source = this.cueCatalog.get(cueId);
+    if (!source) return false;
+    this.pendingCue = oneShotSource(source, DEFAULT_CUE_GAIN);
+    return true;
+  }
+
   /** G3 cue wiring: started/delayed → true, complete/failed → false. */
   setThinking(on: boolean): void {
     if (on) {
       if (this.thinkingState === "on") return;
       this.thinkingState = "on";
       this.thinkingFadeInPos = 0;
-      if (this.thinking) this.thinking.position = 0; // each episode starts from the top
+      if (this.thinking) this.thinking.position = 0;
       return;
     }
     if (this.thinkingState !== "on") return;
-    // Audible for one fade-out ramp, then off (immediately off when fades are disabled).
     this.thinkingState = this.fadeMs > 0 ? "stopping" : "off";
     this.thinkingFadeOutPos = 0;
   }
 
-  /**
-   * Add the bed sources into `mixBuf` at `bedScale`, applying the fade
-   * envelopes and advancing their positions/counters. Shared by mix (ducked)
-   * and idleFrame (full gain) so episodes stay continuous across both.
-   */
+  private thinkingScale(bedScale: number): number {
+    return this.pendingCue ? bedScale * CUE_THINKING_DUCK : bedScale;
+  }
+
   private addBed(mixBuf: Float64Array, sampleRateHz: number, bedScale: number): void {
     const fadeSamples = Math.round((this.fadeMs / 1000) * sampleRateHz);
+    const thinkingScale = this.thinkingScale(bedScale);
 
     if (this.ambient) {
       const startPos = this.ambientFadeInPos;
@@ -201,7 +248,7 @@ export class BackgroundAudioMixer {
         const envelope = startPos >= fadeSamples
           ? undefined
           : (i: number) => fadeInGain(startPos + i, fadeSamples);
-        addLooped(mixBuf, this.thinking, sampleRateHz, this.thinking.gain * bedScale, envelope);
+        addLooped(mixBuf, this.thinking, sampleRateHz, this.thinking.gain * thinkingScale, envelope);
         this.thinkingFadeInPos = Math.min(fadeSamples, startPos + mixBuf.length);
       } else {
         const startPos = this.thinkingFadeOutPos;
@@ -209,7 +256,7 @@ export class BackgroundAudioMixer {
           mixBuf,
           this.thinking,
           sampleRateHz,
-          this.thinking.gain * bedScale,
+          this.thinking.gain * thinkingScale,
           (i: number) => fadeOutGain(startPos + i, fadeSamples),
         );
         this.thinkingFadeOutPos = startPos + mixBuf.length;
@@ -218,46 +265,58 @@ export class BackgroundAudioMixer {
     }
   }
 
-  /**
-   * Mix the background bed (ducked) into an assistant speech chunk and advance
-   * the speaking-until estimate by the chunk's realtime duration. Returns a new
-   * buffer; the input is never mutated.
-   */
+  private addPendingCue(mixBuf: Float64Array, sampleRateHz: number): void {
+    if (!this.pendingCue) return;
+    const finished = addOneShot(mixBuf, this.pendingCue, sampleRateHz);
+    if (finished) this.pendingCue = null;
+  }
+
   mix(chunk: Uint8Array, sampleRateHz: number, nowMs = Date.now()): Uint8Array {
     const speech = pcm16BytesToSamples(chunk);
     const durationMs = (speech.length / sampleRateHz) * 1000;
     this.speakingUntilMs = Math.max(this.speakingUntilMs, nowMs) + durationMs;
 
-    if (!this.hasSources) return chunk;
+    if (!this.hasSources && !this.pendingCue) return chunk;
     const mixBuf = new Float64Array(speech.length);
     for (let i = 0; i < speech.length; i += 1) mixBuf[i] = speech[i]!;
     this.addBed(mixBuf, sampleRateHz, this.duck);
     return clipToPcm16Bytes(mixBuf);
   }
 
-  /**
-   * Bed-only frame for the gaps between turns (telephony comfort noise), or
-   * null while speech is still playing out / there is nothing to play.
-   */
   idleFrame(frameMs: number, sampleRateHz: number, nowMs = Date.now()): Uint8Array | null {
     if (this.isSpeaking(nowMs)) return null;
     const playThinking = this.thinking !== null && this.thinkingState !== "off";
-    if (!this.ambient && !playThinking) return null;
+    if (!this.ambient && !playThinking && !this.pendingCue) return null;
 
     const sampleCount = Math.max(1, Math.round((frameMs / 1000) * sampleRateHz));
     const mixBuf = new Float64Array(sampleCount);
     this.addBed(mixBuf, sampleRateHz, 1);
+    this.addPendingCue(mixBuf, sampleRateHz);
     return clipToPcm16Bytes(mixBuf);
   }
 }
 
-/**
- * Drive the mixer's thinking loop from the session's G3 tool-call cues:
- * started/delayed → thinking on; complete/failed → off. Listener lifetime is
- * the session's — both are per-connection.
- */
 export function wireBackgroundThinking(session: VoiceAgentSession, mixer: BackgroundAudioMixer): void {
   session.on("tool_call_cue", (event) => {
     mixer.setThinking(event.phase === "started" || event.phase === "delayed");
   });
 }
+
+export function wireBackgroundBackchannel(session: VoiceAgentSession, mixer: BackgroundAudioMixer): void {
+  session.bus.on("interaction.backchannel", (pkt) => {
+    const backchannel = pkt as InteractionBackchannelPacket;
+    if (mixer.queueCue(backchannel.cue)) return;
+    session.bus.push(Route.Background, {
+      kind: "metric.conversation",
+      contextId: backchannel.contextId,
+      timestampMs: backchannel.timestampMs,
+      name: "backchannel.suppressed_missing_asset",
+      value: backchannel.cue,
+    });
+  });
+}
+
+export function wireBackgroundAudio(session: VoiceAgentSession, mixer: BackgroundAudioMixer): void {
+  wireBackgroundThinking(session, mixer);
+  wireBackgroundBackchannel(session, mixer);
+}