@kuralle-syrinx/server-websocket 2.1.0

package/src/edge-twilio.ts

@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: MIT
+//
+// Twilio Media Streams ingress for the Workers edge: bridges Twilio's WebSocket
+// protocol (base64 μ-law 8 kHz both ways) to a VoiceAgentSession, mirroring the
+// session-lease/heartbeat pattern of edge.ts. Provider-agnostic — the bridge only
+// speaks core packets (user.audio_received / tts.audio / interrupt.detected).
+//
+// Barge-in: interrupt.detected → Twilio `clear` event, which drops Twilio's
+// buffered playout instantly (their barge-in mechanism). Playout-clock caveat:
+// like the browser edge path, turn-taking uses the estimate fallback (no paced
+// transport progress events on the edge yet).
+
+import {
+  Route,
+  TimerScheduler,
+  type Scheduler,
+  type VoiceAgentSession,
+  type TextToSpeechAudioPacket,
+} from "@kuralle-syrinx/core";
+import {
+  decodeMuLawToPcm16,
+  encodePcm16ToMuLaw,
+  pcm16BytesToSamples,
+  pcm16SamplesToBytes,
+  resamplePcm16Streaming,
+  type StreamingPcm16Resampler,
+} from "@kuralle-syrinx/core/audio";
+import type { ManagedSocket, SocketData } from "@kuralle-syrinx/ws";
+import {
+  createWorkersInboundSocket,
+  type WorkersDurableObjectWebSocketContext,
+} from "@kuralle-syrinx/ws/workers";
+import type { SessionStore } from "./session-store.js";
+
+const TWILIO_SAMPLE_RATE_HZ = 8_000;
+const DEFAULT_RESUME_WINDOW_MS = 15_000;
+const DEFAULT_KEEP_ALIVE_INTERVAL_MS = 15_000;
+const DEFAULT_IDLE_TIMEOUT_MS = 60_000;
+const DEFAULT_STARTUP_TIMEOUT_MS = 15_000;
+const KEEP_ALIVE_KEY = "voice.edge.twilio.keep_alive";
+
+export interface TwilioEdgeWebSocketOptions {
+  readonly sessionStore: SessionStore;
+  readonly createSession: (request: Request) => VoiceAgentSession | Promise<VoiceAgentSession>;
+  readonly scheduler?: Scheduler;
+  /** Engine-side PCM rate (default 16000). Twilio's leg is always 8 kHz μ-law. */
+  readonly engineSampleRateHz?: number;
+  readonly resumeWindowMs?: number;
+  readonly keepAliveIntervalMs?: number;
+  readonly idleTimeoutMs?: number;
+  readonly startupTimeoutMs?: number;
+}
+
+export interface TwilioEdgeWebSocketUpgrade {
+  readonly response: Response;
+  readonly controller: ReturnType<typeof createWorkersInboundSocket>["controller"];
+}
+
+export function createTwilioEdgeWebSocketUpgrade(
+  request: Request,
+  options: TwilioEdgeWebSocketOptions,
+  ctx?: WorkersDurableObjectWebSocketContext,
+): TwilioEdgeWebSocketUpgrade {
+  const inbound = createWorkersInboundSocket(ctx);
+  void runTwilioEdgeWebSocketConnection(inbound.socket, request, options);
+  return { response: inbound.response, controller: inbound.controller };
+}
+
+export async function runTwilioEdgeWebSocketConnection(
+  socket: ManagedSocket,
+  request: Request,
+  options: TwilioEdgeWebSocketOptions,
+): Promise<void> {
+  const scheduler = options.scheduler ?? new TimerScheduler();
+  const engineRate = options.engineSampleRateHz ?? 16_000;
+  const resumeWindowMs = options.resumeWindowMs ?? DEFAULT_RESUME_WINDOW_MS;
+  const keepAliveIntervalMs = options.keepAliveIntervalMs ?? DEFAULT_KEEP_ALIVE_INTERVAL_MS;
+  const idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
+  const startupTimeoutMs = options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
+
+  const uplinkResamplers = new Map<string, StreamingPcm16Resampler>();
+  const downlinkResamplers = new Map<string, StreamingPcm16Resampler>();
+  const disposers: Array<() => void> = [];
+  let session: VoiceAgentSession | null = null;
+  let sessionId = "";
+  let streamSid = "";
+  let contextId = "";
+  let contextBase = "";
+  let turnCounter = 0;
+  let closed = false;
+  let stopped = false;
+  let lastClientMessageMs = Date.now();
+
+  const sendTwilioJson = (value: unknown): void => {
+    if (!socket.isOpen) return;
+    socket.send(JSON.stringify(value));
+  };
+
+  const cleanup = (): void => {
+    if (closed) return;
+    closed = true;
+    scheduler.cancel(KEEP_ALIVE_KEY);
+    for (const dispose of disposers.splice(0)) dispose();
+    if (session && sessionId) {
+      void options.sessionStore.release(sessionId, stopped ? 0 : resumeWindowMs);
+    }
+  };
+
+  socket.onClose(() => {
+    cleanup();
+  });
+  socket.onError(() => {
+    cleanup();
+  });
+
+  // Twilio sends connected/start (and media) immediately after the upgrade —
+  // before the session lease (provider sockets, kuralle init) resolves. Buffer
+  // until the handler is live or the start event is lost and the whole call
+  // streams into the void.
+  const pendingMessages: Array<{ data: SocketData; isBinary: boolean }> = [];
+  let pendingBytes = 0;
+  let liveHandler: ((data: SocketData, isBinary: boolean) => void) | null = null;
+  socket.onMessage((data: SocketData, isBinary: boolean) => {
+    lastClientMessageMs = Date.now();
+    if (liveHandler) {
+      liveHandler(data, isBinary);
+      return;
+    }
+    const byteLength = typeof data === "string" ? data.length : (data as ArrayBuffer | Uint8Array).byteLength;
+    if (pendingBytes + byteLength > 2 * 1024 * 1024) return; // cap startup buffering
+    pendingBytes += byteLength;
+    pendingMessages.push({ data, isBinary });
+  });
+
+  try {
+    const url = new URL(request.url);
+    sessionId = url.searchParams.get("sessionId")?.trim() || crypto.randomUUID();
+
+    const startupTimer = new Promise<never>((_, reject) => {
+      scheduler.schedule("voice.edge.twilio.startup", startupTimeoutMs, () => {
+        reject(new Error("twilio session startup timeout"));
+      });
+    });
+    const leased = await Promise.race([
+      options.sessionStore.lease(sessionId, async () => {
+        const sess = await options.createSession(request);
+        await sess.start();
+        return {
+          id: sessionId,
+          session: sess,
+          currentContextId: "",
+          contextSampleRates: new Map(),
+          inputSequence: { lastSequence: null },
+          turnMetricsTurns: new Map(),
+          closeTimer: null,
+          connectionCount: 1,
+        };
+      }),
+      startupTimer,
+    ]);
+    scheduler.cancel("voice.edge.twilio.startup");
+    session = leased.managed.session;
+    if (closed) {
+      await options.sessionStore.release(sessionId, 0);
+      return;
+    }
+
+    // Downlink: engine PCM → 8 kHz μ-law media frames; barge-in → clear.
+    disposers.push(
+      session.bus.on("tts.audio", (pkt) => {
+        if (!streamSid) return;
+        const audio = pkt as TextToSpeechAudioPacket;
+        const samples = pcm16BytesToSamples(audio.audio);
+        const sourceRate = audio.sampleRateHz ?? engineRate;
+        const resampled = resamplePcm16Streaming(downlinkResamplers, samples, sourceRate, TWILIO_SAMPLE_RATE_HZ);
+        const mulaw = encodePcm16ToMuLaw(resampled);
+        sendTwilioJson({
+          event: "media",
+          streamSid,
+          media: { payload: bytesToBase64(mulaw) },
+        });
+      }),
+      session.bus.on("interrupt.detected", () => {
+        if (!streamSid) return;
+        sendTwilioJson({ event: "clear", streamSid });
+      }),
+      // The phone mic streams continuously on one call, but the engine finishes a
+      // turn per contextId — rotate on turn_complete exactly like the browser
+      // client does, or only the first utterance is ever heard (the STT plugin
+      // drops transcripts for already-finalized contexts).
+      session.bus.on("eos.turn_complete", () => {
+        if (!contextBase) return;
+        turnCounter += 1;
+        contextId = `${contextBase}-t${String(turnCounter)}`;
+      }),
+    );
+
+    if (keepAliveIntervalMs > 0) {
+      const heartbeat = (): void => {
+        if (closed) return;
+        if (idleTimeoutMs > 0 && Date.now() - lastClientMessageMs > idleTimeoutMs) {
+          socket.dispose();
+          return;
+        }
+        scheduler.schedule(KEEP_ALIVE_KEY, keepAliveIntervalMs, heartbeat);
+      };
+      scheduler.schedule(KEEP_ALIVE_KEY, keepAliveIntervalMs, heartbeat);
+    }
+
+    const handleMessage = (data: SocketData, isBinary: boolean): void => {
+      if (isBinary || closed || !session) return;
+      let message: Record<string, unknown>;
+      try {
+        message = JSON.parse(socketDataToText(data)) as Record<string, unknown>;
+      } catch {
+        return;
+      }
+      const event = message["event"];
+      if (event === "connected" || event === "mark" || event === "dtmf") return;
+      if (event === "start") {
+        const start = (message["start"] ?? {}) as Record<string, unknown>;
+        streamSid =
+          (typeof start["streamSid"] === "string" && start["streamSid"]) ||
+          (typeof message["streamSid"] === "string" && message["streamSid"]) ||
+          "";
+        const callSid = typeof start["callSid"] === "string" ? start["callSid"] : "";
+        contextBase = callSid ? `twilio-${callSid}` : `twilio-${streamSid || sessionId}`;
+        contextId = contextBase;
+        return;
+      }
+      if (event === "media") {
+        const media = (message["media"] ?? {}) as Record<string, unknown>;
+        const payload = typeof media["payload"] === "string" ? media["payload"] : "";
+        if (!payload || !contextId) return;
+        const mulaw = base64ToBytes(payload);
+        const pcm8k = decodeMuLawToPcm16(mulaw);
+        const pcm16k = resamplePcm16Streaming(uplinkResamplers, pcm8k, TWILIO_SAMPLE_RATE_HZ, engineRate);
+        session.bus.push(Route.Main, {
+          kind: "user.audio_received",
+          contextId,
+          timestampMs: Date.now(),
+          audio: pcm16SamplesToBytes(pcm16k),
+        });
+        return;
+      }
+      if (event === "stop") {
+        stopped = true;
+        socket.dispose();
+      }
+    };
+    liveHandler = handleMessage;
+    for (const queued of pendingMessages.splice(0)) {
+      handleMessage(queued.data, queued.isBinary);
+    }
+  } catch (err) {
+    sendTwilioJson({
+      event: "error",
+      message: err instanceof Error ? err.message : String(err),
+    });
+    socket.dispose();
+    cleanup();
+  }
+}
+
+function socketDataToText(data: SocketData): string {
+  if (typeof data === "string") return data;
+  if (data instanceof ArrayBuffer) return new TextDecoder().decode(new Uint8Array(data));
+  return new TextDecoder().decode(data as Uint8Array);
+}
+
+function base64ToBytes(value: string): Uint8Array {
+  const binary = atob(value);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i);
+  return bytes;
+}
+
+function bytesToBase64(bytes: Uint8Array): string {
+  let binary = "";
+  const chunkSize = 0x8000;
+  for (let offset = 0; offset < bytes.length; offset += chunkSize) {
+    binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
+  }
+  return btoa(binary);
+}

package/src/edge.test.ts

@@ -0,0 +1,272 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+import {
+  PipelineBusImpl,
+  Route,
+  encodeSyrinxAudioEnvelope,
+  type Scheduler,
+  type ScheduledCallback,
+  type UserAudioReceivedPacket,
+  type VoiceAgentSession,
+} from "@kuralle-syrinx/core";
+import { pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
+import type { ManagedSocket, SocketData } from "@kuralle-syrinx/ws";
+import { InMemorySessionStore } from "./session-store.js";
+import { runVoiceEdgeWebSocketConnection } from "./edge.js";
+
+const KEEP_ALIVE_KEY = "voice.edge.keep_alive";
+
+/** Scheduler whose tasks fire only when the test calls fire() — no real timers. */
+class ManualScheduler implements Scheduler {
+  readonly tasks = new Map<string, ScheduledCallback>();
+  schedule(key: string, _delayMs: number, cb: ScheduledCallback): void {
+    this.tasks.set(key, cb);
+  }
+  cancel(key: string): void {
+    this.tasks.delete(key);
+  }
+  async fire(key: string): Promise<void> {
+    const cb = this.tasks.get(key);
+    this.tasks.delete(key);
+    if (cb) await cb();
+  }
+}
+
+class FakeSocket implements ManagedSocket {
+  isOpen = true;
+  disposed = false;
+  readonly sent: SocketData[] = [];
+  #onMessage?: (data: SocketData, isBinary: boolean) => void;
+  #onClose?: (code: number, reason: string) => void;
+  get isOpenValue(): boolean {
+    return this.isOpen;
+  }
+  send(data: SocketData): void {
+    this.sent.push(data);
+  }
+  keepAlivePing(): void {}
+  async verify(): Promise<boolean> {
+    return this.isOpen;
+  }
+  dispose(): void {
+    this.disposed = true;
+    this.isOpen = false;
+    this.#onClose?.(1000, "disposed");
+  }
+  onOpen(): void {}
+  onMessage(handler: (data: SocketData, isBinary: boolean) => void): void {
+    this.#onMessage = handler;
+  }
+  onClose(handler: (code: number, reason: string) => void): void {
+    this.#onClose = handler;
+  }
+  onError(): void {}
+  emit(data: SocketData, isBinary = false): void {
+    this.#onMessage?.(data, isBinary);
+  }
+  json(): Array<Record<string, unknown>> {
+    return this.sent
+      .filter((s): s is string => typeof s === "string")
+      .map((s) => JSON.parse(s) as Record<string, unknown>);
+  }
+}
+
+function fakeSession(received: UserAudioReceivedPacket[] = []): VoiceAgentSession {
+  const bus = new PipelineBusImpl();
+  bus.on("user.audio_received", (pkt) => {
+    received.push(pkt as UserAudioReceivedPacket);
+  });
+  return {
+    bus,
+    async start() {
+      void bus.start();
+    },
+    async close() {
+      bus.stop();
+    },
+    on() {},
+    off() {},
+    requestClientInterrupt() {},
+  } as unknown as VoiceAgentSession;
+}
+
+function runConnection(
+  socket: FakeSocket,
+  scheduler: ManualScheduler,
+  opts: {
+    idleTimeoutMs?: number;
+    keepAliveIntervalMs?: number;
+    rawBinaryInput?: boolean;
+    received?: UserAudioReceivedPacket[];
+  } = {},
+) {
+  const received = opts.received ?? [];
+  return runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+    sessionStore: new InMemorySessionStore(),
+    scheduler,
+    createSession: () => fakeSession(received),
+    idleTimeoutMs: opts.idleTimeoutMs,
+    keepAliveIntervalMs: opts.keepAliveIntervalMs,
+    rawBinaryInput: opts.rawBinaryInput,
+  });
+}
+
+function waitForReady(socket: FakeSocket): void {
+  expect(socket.json().some((m) => m.type === "ready")).toBe(true);
+}
+
+function encodeEnvelopeFrame(
+  sequence: number,
+  audio: Uint8Array,
+  contextId = "turn-envelope",
+): Uint8Array {
+  return encodeSyrinxAudioEnvelope({
+    type: "audio",
+    contextId,
+    sampleRateHz: 16000,
+    encoding: "pcm_s16le",
+    channels: 1,
+    byteLength: audio.byteLength,
+    sequence,
+  }, audio);
+}
+
+function findOddAndEvenEnvelopeFrames(
+  pcm: Uint8Array,
+): { ordered: [Uint8Array, Uint8Array]; odd: Uint8Array; even: Uint8Array } {
+  const candidates: Array<{ frame: Uint8Array; parity: number; sequence: number }> = [];
+  for (let index = 0; index < 32; index += 1) {
+    const frame = encodeEnvelopeFrame(index + 1, pcm, `turn-envelope-${"x".repeat(index)}`);
+    candidates.push({ frame, parity: frame.byteLength % 2, sequence: index + 1 });
+  }
+  const odd = candidates.find((entry) => entry.parity === 1);
+  const even = candidates.find((entry) => entry.parity === 0 && entry.sequence > (odd?.sequence ?? 0));
+  if (!odd || !even) throw new Error("Could not construct odd and even envelope frame lengths");
+  const ordered = odd.sequence < even.sequence
+    ? [odd.frame, even.frame] as [Uint8Array, Uint8Array]
+    : [even.frame, odd.frame] as [Uint8Array, Uint8Array];
+  return { ordered, odd: odd.frame, even: even.frame };
+}
+
+describe("edge keep-alive / idle close", () => {
+  it("arms a keep-alive heartbeat once the session is ready and re-arms while active", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runConnection(socket, scheduler, { idleTimeoutMs: 10_000, keepAliveIntervalMs: 1_000 });
+
+    expect(socket.json().some((m) => m.type === "ready")).toBe(true);
+    expect(scheduler.tasks.has(KEEP_ALIVE_KEY)).toBe(true);
+
+    // A fresh heartbeat with recent client activity must re-arm, not close.
+    await scheduler.fire(KEEP_ALIVE_KEY);
+    expect(socket.disposed).toBe(false);
+    expect(scheduler.tasks.has(KEEP_ALIVE_KEY)).toBe(true);
+  });
+
+  it("closes a half-open client that has been idle past idleTimeoutMs", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runConnection(socket, scheduler, { idleTimeoutMs: 30, keepAliveIntervalMs: 1_000 });
+
+    // No further client traffic; let the idle window elapse, then beat.
+    await new Promise((r) => setTimeout(r, 60));
+    await scheduler.fire(KEEP_ALIVE_KEY);
+
+    expect(socket.disposed).toBe(true);
+    expect(socket.json().some((m) => m.type === "error" && m.category === "idle_timeout")).toBe(true);
+  });
+
+  it("a client message resets the idle window", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runConnection(socket, scheduler, { idleTimeoutMs: 30, keepAliveIntervalMs: 1_000 });
+
+    await new Promise((r) => setTimeout(r, 60));
+    socket.emit(JSON.stringify({ type: "ping" })); // refreshes lastClientMessageMs
+    await scheduler.fire(KEEP_ALIVE_KEY);
+
+    expect(socket.disposed).toBe(false);
+    expect(scheduler.tasks.has(KEEP_ALIVE_KEY)).toBe(true);
+  });
+
+  it("keepAliveIntervalMs=0 disables the heartbeat entirely", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runConnection(socket, scheduler, { keepAliveIntervalMs: 0 });
+
+    expect(socket.json().some((m) => m.type === "ready")).toBe(true);
+    expect(scheduler.tasks.has(KEEP_ALIVE_KEY)).toBe(false);
+  });
+});
+
+describe("edge inbound binary audio envelopes", () => {
+  it("decodes syrinx.audio.v1 envelopes into even PCM payloads", async () => {
+    const received: UserAudioReceivedPacket[] = [];
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runConnection(socket, scheduler, { received });
+    waitForReady(socket);
+
+    const pcm = pcm16SamplesToBytes(new Int16Array([0, 32767, -32768, 16384]));
+    const { ordered, odd: oddHeaderFrame, even: evenHeaderFrame } = findOddAndEvenEnvelopeFrames(pcm);
+    expect(oddHeaderFrame.byteLength % 2).toBe(1);
+    expect(evenHeaderFrame.byteLength % 2).toBe(0);
+
+    for (const frame of ordered) socket.emit(frame, true);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(received).toHaveLength(2);
+    for (const packet of received) {
+      expect(packet.kind).toBe("user.audio_received");
+      expect(packet.contextId?.startsWith("turn-envelope")).toBe(true);
+      expect(packet.audio.byteLength % 2).toBe(0);
+      expect(packet.audio).toEqual(pcm);
+    }
+    expect(socket.json().some((m) => m.type === "error")).toBe(false);
+  });
+
+  it("rejects raw binary audio unless rawBinaryInput is enabled", async () => {
+    const received: UserAudioReceivedPacket[] = [];
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    await runConnection(socket, scheduler, { received });
+    waitForReady(socket);
+
+    socket.emit(new Uint8Array([1, 2, 3, 4]), true);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(received).toEqual([]);
+    expect(socket.json()).toContainEqual({
+      type: "error",
+      component: "transport",
+      category: "invalid_input",
+      message: "Raw binary websocket audio is disabled; use syrinx.audio.v1 or JSON audio frames",
+    });
+  });
+});
+
+describe("edge barge-in downlink", () => {
+  it("sends audio_clear and agent_interrupted when an interrupt is detected", async () => {
+    const socket = new FakeSocket();
+    const scheduler = new ManualScheduler();
+    const session = fakeSession();
+    await runVoiceEdgeWebSocketConnection(socket, new Request("https://edge.test/ws?sessionId=s1"), {
+      sessionStore: new InMemorySessionStore(),
+      scheduler,
+      createSession: () => session,
+    });
+    waitForReady(socket);
+
+    session.bus.push(Route.Critical, {
+      kind: "interrupt.detected",
+      contextId: "assistant-turn",
+      timestampMs: Date.now(),
+      source: "vad",
+    });
+    await new Promise((resolve) => setTimeout(resolve, 0));
+
+    expect(socket.json()).toContainEqual({ type: "audio_clear", turnId: "assistant-turn", reason: "barge_in" });
+    expect(socket.json()).toContainEqual({ type: "agent_interrupted", turnId: "assistant-turn", reason: "barge_in" });
+  });
+});