@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022329.d89bf2f

package/src/__tests__/media-stream-output.test.ts

@@ -29,6 +29,7 @@ mock.module("../calls/resolve-call-tts-provider.js", () => ({
   })),
 }));
 
+import { mulawToPcm16 } from "../calls/media-stream-audio-transcode.js";
 import { MediaStreamOutput } from "../calls/media-stream-output.js";
 import { resolveCallTtsProvider } from "../calls/resolve-call-tts-provider.js";
 
@@ -84,7 +85,12 @@ async function drain(): Promise<void> {
 }
 
 /** Generate a minimal valid WAV buffer with PCM data. */
-function makeWavBuffer(pcmSamples: number[]): Buffer {
+function makeWavBuffer(
+  pcmSamples: number[],
+  opts?: { sampleRate?: number; channels?: number },
+): Buffer {
+  const sampleRate = opts?.sampleRate ?? 8000;
+  const channels = opts?.channels ?? 1;
   const pcmData = Buffer.alloc(pcmSamples.length * 2);
   for (let i = 0; i < pcmSamples.length; i++) {
     pcmData.writeInt16LE(pcmSamples[i], i * 2);
@@ -97,16 +103,60 @@ function makeWavBuffer(pcmSamples: number[]): Buffer {
   header.write("fmt ", 12);
   header.writeUInt32LE(16, 16); // subchunk1 size
   header.writeUInt16LE(1, 20); // PCM
-  header.writeUInt16LE(1, 22); // mono
-  header.writeUInt32LE(8000, 24); // sample rate
-  header.writeUInt32LE(16000, 28); // byte rate
-  header.writeUInt16LE(2, 32); // block align
+  header.writeUInt16LE(channels, 22);
+  header.writeUInt32LE(sampleRate, 24);
+  header.writeUInt32LE(sampleRate * channels * 2, 28); // byte rate
+  header.writeUInt16LE(channels * 2, 32); // block align
   header.writeUInt16LE(16, 34); // bits per sample
   header.write("data", 36);
   header.writeUInt32LE(pcmData.length, 40);
   return Buffer.concat([header, pcmData]);
 }
 
+/** Generate `count` samples of a sine tone at `freqHz` for a given rate. */
+function sineSamples(
+  count: number,
+  freqHz: number,
+  sampleRate: number,
+): number[] {
+  return Array.from({ length: count }, (_, i) =>
+    Math.round(Math.sin((2 * Math.PI * freqHz * i) / sampleRate) * 10000),
+  );
+}
+
+/** Total decoded mu-law byte count across all sent media frames. */
+function totalMulawBytes(sent: string[]): number {
+  return sent
+    .filter((s) => JSON.parse(s).event === "media")
+    .reduce(
+      (sum, s) =>
+        sum + Buffer.from(JSON.parse(s).media.payload, "base64").length,
+      0,
+    );
+}
+
+/** Concatenate the decoded mu-law payloads of all sent media frames. */
+function concatMulawPayloads(sent: string[]): Buffer {
+  return Buffer.concat(
+    sent
+      .filter((s) => JSON.parse(s).event === "media")
+      .map((s) => Buffer.from(JSON.parse(s).media.payload, "base64")),
+  );
+}
+
+/** Count sign changes in a PCM16 LE buffer (proxy for tone frequency). */
+function countZeroCrossings(pcm: Buffer): number {
+  const samples = Math.floor(pcm.length / 2);
+  let crossings = 0;
+  let prev = pcm.readInt16LE(0);
+  for (let i = 1; i < samples; i++) {
+    const s = pcm.readInt16LE(i * 2);
+    if ((prev < 0 && s >= 0) || (prev >= 0 && s < 0)) crossings++;
+    prev = s;
+  }
+  return crossings;
+}
+
 // ---------------------------------------------------------------------------
 // Setup / teardown
 // ---------------------------------------------------------------------------
@@ -230,19 +280,6 @@ describe("MediaStreamOutput", () => {
       output.endSession("second");
       expect(mock.closed).toBe(true);
     });
-
-    test("getConnectionState returns 'connected' initially", () => {
-      const { ws } = createMockWs();
-      const output = new MediaStreamOutput(ws, "stream-1");
-      expect(output.getConnectionState()).toBe("connected");
-    });
-
-    test("getConnectionState returns 'closed' after endSession", () => {
-      const mock = createMockWs();
-      const output = new MediaStreamOutput(mock.ws, "stream-1");
-      output.endSession();
-      expect(output.getConnectionState()).toBe("closed");
-    });
   });
 
   describe("sendAudioPayload", () => {
@@ -349,6 +386,224 @@ describe("MediaStreamOutput", () => {
     });
   });
 
+  describe("clearBufferedAudio — rejected barge-in", () => {
+    test("sends a clear command to Twilio", () => {
+      const { ws, sent } = createMockWs();
+      const output = new MediaStreamOutput(ws, "MZ-stream-1");
+      output.clearBufferedAudio();
+
+      expect(sent).toHaveLength(1);
+      expect(JSON.parse(sent[0])).toEqual({
+        event: "clear",
+        streamSid: "MZ-stream-1",
+      });
+    });
+
+    test("preserves in-flight synthesis — queued speech still plays", async () => {
+      const wav = makeWavBuffer([1000, 2000, 3000, 4000]);
+      // Slow synthesis so the clear lands while the item is in flight
+      mockSynthesize.mockImplementation(
+        () =>
+          new Promise((resolve) =>
+            setTimeout(
+              () => resolve({ audio: wav, contentType: "audio/wav" }),
+              20,
+            ),
+          ),
+      );
+
+      const { ws, sent } = createMockWs();
+      const output = new MediaStreamOutput(ws, "MZ-stream-1");
+
+      output.sendTextToken("hello world", true);
+      output.clearBufferedAudio();
+
+      await new Promise((resolve) => setTimeout(resolve, 60));
+
+      // Unlike clearAudio, the in-flight synthesis is not aborted:
+      // its frames go out once ready.
+      const mediaMessages = sent.filter((s) => JSON.parse(s).event === "media");
+      expect(mediaMessages.length).toBeGreaterThan(0);
+    });
+
+    test("keeps the audio-start signal armed", async () => {
+      const wav = makeWavBuffer([1000, 2000, 3000, 4000]);
+      mockSynthesize.mockImplementation(
+        () =>
+          new Promise((resolve) =>
+            setTimeout(
+              () => resolve({ audio: wav, contentType: "audio/wav" }),
+              20,
+            ),
+          ),
+      );
+
+      const { ws } = createMockWs();
+      const output = new MediaStreamOutput(ws, "MZ-stream-1");
+
+      let fired = 0;
+      output.setAudioStartCallback(() => {
+        fired++;
+      });
+      output.sendTextToken("hello world", true);
+      output.clearBufferedAudio();
+
+      await new Promise((resolve) => setTimeout(resolve, 60));
+
+      expect(fired).toBe(1);
+    });
+
+    test("does not send when closed", () => {
+      const { ws, sent } = createMockWs();
+      const output = new MediaStreamOutput(ws, "MZ-stream-1");
+      output.endSession();
+      output.clearBufferedAudio();
+      expect(sent).toHaveLength(0);
+    });
+  });
+
+  describe("audio-start signal", () => {
+    function makePlayableWav(): Buffer {
+      const samples = Array.from({ length: 400 }, (_, i) =>
+        Math.round(Math.sin(i * 0.1) * 10000),
+      );
+      return makeWavBuffer(samples);
+    }
+
+    test("fires once when the first audio frame of a synthesize item is sent", async () => {
+      mockSynthesize.mockResolvedValue({
+        audio: makePlayableWav(),
+        contentType: "audio/wav",
+      });
+
+      const { ws, sent } = createMockWs();
+      const output = new MediaStreamOutput(ws, "stream-1");
+
+      let fired = 0;
+      let mediaSentWhenFired = -1;
+      output.setAudioStartCallback(() => {
+        fired++;
+        mediaSentWhenFired = sent.filter(
+          (s) => JSON.parse(s).event === "media",
+        ).length;
+      });
+
+      output.sendTextToken("hello world", true);
+      await drain();
+
+      const mediaMessages = sent.filter((s) => JSON.parse(s).event === "media");
+      expect(mediaMessages.length).toBeGreaterThan(1);
+      // Fired exactly once, before any media frame went out.
+      expect(fired).toBe(1);
+      expect(mediaSentWhenFired).toBe(0);
+    });
+
+    test("one-shot: does not fire again for a subsequent item until re-armed", async () => {
+      mockSynthesize.mockResolvedValue({
+        audio: makePlayableWav(),
+        contentType: "audio/wav",
+      });
+
+      const { ws } = createMockWs();
+      const output = new MediaStreamOutput(ws, "stream-1");
+
+      let fired = 0;
+      output.setAudioStartCallback(() => fired++);
+
+      output.sendTextToken("first", true);
+      await drain();
+      expect(fired).toBe(1);
+
+      // Second item without re-arming — signal already consumed.
+      output.sendTextToken("second", true);
+      await drain();
+      expect(fired).toBe(1);
+
+      // Re-armed — fires again for the next item.
+      output.setAudioStartCallback(() => fired++);
+      output.sendTextToken("third", true);
+      await drain();
+      expect(fired).toBe(2);
+    });
+
+    test("cleared on clearAudio: flushed playback never fires the signal", async () => {
+      // Slow synthesis so clearAudio lands while the item is in flight.
+      mockSynthesize.mockImplementation(
+        () =>
+          new Promise((resolve) =>
+            setTimeout(
+              () =>
+                resolve({ audio: makePlayableWav(), contentType: "audio/wav" }),
+              200,
+            ),
+          ),
+      );
+
+      const { ws } = createMockWs();
+      const output = new MediaStreamOutput(ws, "stream-1");
+
+      let fired = 0;
+      output.setAudioStartCallback(() => fired++);
+
+      output.sendTextToken("interrupted response", true);
+      output.clearAudio();
+
+      await new Promise((resolve) => setTimeout(resolve, 300));
+      expect(fired).toBe(0);
+    });
+
+    test("an empty end-of-turn (mark only) does not fire the signal", async () => {
+      const { ws } = createMockWs();
+      const output = new MediaStreamOutput(ws, "stream-1");
+
+      let fired = 0;
+      output.setAudioStartCallback(() => fired++);
+
+      output.sendTextToken("", true);
+      await drain();
+      expect(fired).toBe(0);
+    });
+  });
+
+  describe("buffered text lifecycle", () => {
+    test("clearAudio preserves text accumulating for an in-flight turn", async () => {
+      const wav = makeWavBuffer([1000, 2000, 3000, 4000]);
+      mockSynthesize.mockResolvedValue({
+        audio: wav,
+        contentType: "audio/wav",
+      });
+
+      const { ws } = createMockWs();
+      const output = new MediaStreamOutput(ws, "stream-1");
+
+      // Tokens buffered mid-turn; a barge-in signal the controller ignores
+      // (turn still processing) flushes queued audio but must not truncate
+      // the pending response text.
+      output.sendTextToken("hello ", false);
+      output.clearAudio();
+      output.sendTextToken("world", true);
+      await drain();
+
+      expect(mockSynthesize).toHaveBeenCalledTimes(1);
+      expect(mockSynthesize.mock.calls[0][0].text).toBe("hello world");
+    });
+
+    test("discardPendingText drops accumulated text so no synthesis occurs", async () => {
+      const { ws, sent } = createMockWs();
+      const output = new MediaStreamOutput(ws, "stream-1");
+
+      output.sendTextToken("stale partial response", false);
+      output.discardPendingText();
+      output.sendTextToken("", true);
+      await drain();
+
+      expect(mockSynthesize).not.toHaveBeenCalled();
+      // Only the end-of-turn mark goes out.
+      expect(sent).toHaveLength(1);
+      expect(JSON.parse(sent[0]).event).toBe("mark");
+    });
+  });
+
   describe("setStreamSid / getStreamSid", () => {
     test("updates the stream SID used in subsequent commands", () => {
       const { ws, sent } = createMockWs();
@@ -369,7 +624,6 @@ describe("MediaStreamOutput", () => {
       const mock = createMockWs();
       const output = new MediaStreamOutput(mock.ws, "stream-1");
       output.markClosed();
-      expect(output.getConnectionState()).toBe("closed");
       expect(mock.closed).toBe(false); // WebSocket not actually closed
       output.sendAudioPayload("dGVzdA=="); // Should be suppressed
       expect(mock.sent).toHaveLength(0);
@@ -552,4 +806,81 @@ describe("MediaStreamOutput", () => {
       }
     });
   });
+
+  // ---------------------------------------------------------------------------
+  // Regression: WAV sample-rate handling (Fish Audio defaults WAV to 44.1 kHz)
+  // ---------------------------------------------------------------------------
+
+  describe("WAV sample-rate handling", () => {
+    async function synthesizeWav(wav: Buffer): Promise<string[]> {
+      mockSynthesize.mockResolvedValue({
+        audio: wav,
+        contentType: "audio/wav",
+      });
+      const { ws, sent } = createMockWs();
+      const output = new MediaStreamOutput(ws, "stream-1");
+      output.sendTextToken("test", true);
+      await drain();
+      return sent;
+    }
+
+    test("8 kHz WAV passes through without rate conversion", async () => {
+      const wav = makeWavBuffer(sineSamples(400, 440, 8000), {
+        sampleRate: 8000,
+      });
+      const sent = await synthesizeWav(wav);
+      // 400 samples at 8 kHz -> 400 mu-law bytes, unchanged.
+      expect(totalMulawBytes(sent)).toBe(400);
+    });
+
+    test("16 kHz WAV is downsampled by 2 to 8 kHz", async () => {
+      const wav = makeWavBuffer(sineSamples(800, 440, 16000), {
+        sampleRate: 16000,
+      });
+      const sent = await synthesizeWav(wav);
+      // 800 samples at 16 kHz -> 400 mu-law bytes after decimation.
+      expect(totalMulawBytes(sent)).toBe(400);
+    });
+
+    test("44.1 kHz WAV is resampled to 8 kHz preserving pitch", async () => {
+      // 4410 samples = 100 ms of a 440 Hz tone at Fish Audio's WAV default.
+      const wav = makeWavBuffer(sineSamples(4410, 440, 44100), {
+        sampleRate: 44100,
+      });
+      const sent = await synthesizeWav(wav);
+
+      // 100 ms at 8 kHz = 800 mu-law bytes (allow ±1 frame of tolerance).
+      const total = totalMulawBytes(sent);
+      expect(Math.abs(total - 800)).toBeLessThanOrEqual(160);
+
+      // Quality check: decode back to PCM and verify the tone frequency
+      // survived. A 440 Hz tone over 100 ms has ~88 zero crossings; a
+      // 5.5x-slow playback bug would show ~16 instead.
+      const decoded = mulawToPcm16(concatMulawPayloads(sent));
+      const crossings = countZeroCrossings(decoded);
+      expect(Math.abs(crossings - 88)).toBeLessThanOrEqual(4);
+    });
+
+    test("unparseable (zero) sample rate falls back to the 8 kHz assumption", async () => {
+      const wav = makeWavBuffer(sineSamples(400, 440, 8000), {
+        sampleRate: 0,
+      });
+      const sent = await synthesizeWav(wav);
+      // Falls back to current behavior: samples pass through 1:1.
+      expect(totalMulawBytes(sent)).toBe(400);
+    });
+
+    test("stereo WAV keeps one channel", async () => {
+      // 400 interleaved L/R sample pairs (800 samples total) at 8 kHz.
+      const left = sineSamples(400, 440, 8000);
+      const interleaved: number[] = [];
+      for (const s of left) {
+        interleaved.push(s, -s);
+      }
+      const wav = makeWavBuffer(interleaved, { sampleRate: 8000, channels: 2 });
+      const sent = await synthesizeWav(wav);
+      // One channel of 400 samples -> 400 mu-law bytes.
+      expect(totalMulawBytes(sent)).toBe(400);
+    });
+  });
 });