@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts

@@ -11,6 +11,7 @@ import type {
   StreamingTranscriber,
   SttStreamServerEvent,
 } from "../../stt/types.js";
+import { LiveActivityReporter } from "../live-activity-reporter.js";
 import {
   LiveVoiceSession,
   type LiveVoiceTtsStreamer,
@@ -61,11 +62,29 @@ class MockStreamingTranscriber implements StreamingTranscriber {
   }
 }
 
+class RecordingLiveActivityReporter extends LiveActivityReporter {
+  readonly dispatched: Array<{ phase: string; event: string; detail: string }> =
+    [];
+
+  waitForDispatches(): Promise<void> {
+    return this.waitForPendingDispatches();
+  }
+
+  protected override async dispatch(
+    phase: string,
+    event: "update" | "end",
+    detail: string,
+  ): Promise<void> {
+    this.dispatched.push({ phase, event, detail });
+  }
+}
+
 function createHarness(
   startVoiceTurn: LiveVoiceTurnStarter,
   opts: {
     transcriber?: MockStreamingTranscriber;
     streamTtsAudio?: LiveVoiceTtsStreamer;
+    liveActivityReporter?: LiveActivityReporter;
   } = {},
 ) {
   const sequencer = createLiveVoiceServerFrameSequencer();
@@ -84,6 +103,9 @@ function createHarness(
     resolveTranscriber: mock(async () => transcriber),
     startVoiceTurn,
     ...(opts.streamTtsAudio ? { streamTtsAudio: opts.streamTtsAudio } : {}),
+    ...(opts.liveActivityReporter
+      ? { liveActivityReporter: opts.liveActivityReporter }
+      : {}),
     createTurnId: () => "live-turn-1",
     emitMetrics: false,
   });
@@ -102,11 +124,20 @@ function scriptedStartVoiceTurn(script: {
   // Leave the escalated leg in flight (no deltas, no completion) so a barge-in
   // has a live turn to abort mid-hand-off.
   holdEscalated?: boolean;
+  // The escalation judge's verdict, carried on the front-door handle.
+  judgeVerdict?: Promise<boolean>;
 }) {
   const frontDoorAbort = mock();
+  const frontDoorOverrule = mock();
   const escalatedAbort = mock();
   const starter = mock(async (options: VoiceTurnOptions) => {
     const isEscalated = options.content === ESCALATION_CONTINUATION_CONTENT;
+    if (isEscalated) {
+      options.onEscalationTargetResolved?.({
+        profile: "quality-optimized",
+        source: "conversation",
+      });
+    }
     if (isEscalated && script.holdEscalated) {
       return { turnId: "bridge-escalated", abort: escalatedAbort };
     }
@@ -130,9 +161,15 @@ function scriptedStartVoiceTurn(script: {
     return {
       turnId: isEscalated ? "bridge-escalated" : "bridge-front-door",
       abort: isEscalated ? escalatedAbort : frontDoorAbort,
+      ...(!isEscalated && script.judgeVerdict
+        ? {
+            escalationJudgement: script.judgeVerdict,
+            overrule: frontDoorOverrule,
+          }
+        : {}),
     };
   });
-  return { starter, frontDoorAbort, escalatedAbort };
+  return { starter, frontDoorAbort, frontDoorOverrule, escalatedAbort };
 }
 
 async function driveTurn(session: LiveVoiceSession): Promise<void> {
@@ -200,6 +237,150 @@ describe("live-voice triage-and-escalate routing", () => {
     expect(escalated?.overrideProfile).toBeUndefined();
     expect(escalated?.routingLeg).toBe("escalated");
     expect(escalated?.content).toBe(ESCALATION_CONTINUATION_CONTENT);
+    expect(frames).toContainEqual(
+      expect.objectContaining({
+        type: "activity",
+        kind: "escalation",
+        label: "",
+        profile: "quality-optimized",
+        profileSource: "conversation",
+      }),
+    );
+  });
+
+  test("restores the waiting activity after late bridge audio", async () => {
+    const { starter } = scriptedStartVoiceTurn({
+      frontDoor: ["[1] ", "Let me think about that."],
+      holdEscalated: true,
+    });
+    const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
+      options.onAudioChunk({
+        type: "tts_audio",
+        contentType: "audio/pcm",
+        sampleRate: 24_000,
+        dataBase64: "AA==",
+      });
+      return {
+        provider: "fish-audio" as const,
+        contentType: "audio/pcm",
+        sampleRate: 24_000,
+        chunks: 1,
+        bytes: 1,
+      };
+    });
+    const { frames, session } = createHarness(starter, { streamTtsAudio });
+
+    await driveTurn(session);
+    await waitFor(() => starter.mock.calls.length >= 2);
+    await waitFor(
+      () =>
+        frames.filter(
+          (frame) => frame.type === "activity" && frame.kind === "escalation",
+        ).length >= 2,
+    );
+
+    const bridgeAudioIndex = frames.findIndex(
+      (frame) => frame.type === "tts_audio",
+    );
+    const restoredActivityIndex = frames.findIndex(
+      (frame, index) =>
+        index > bridgeAudioIndex &&
+        frame.type === "activity" &&
+        frame.kind === "escalation",
+    );
+    expect(bridgeAudioIndex).toBeGreaterThanOrEqual(0);
+    expect(restoredActivityIndex).toBeGreaterThan(bridgeAudioIndex);
+
+    await session.handleClientFrame({ type: "interrupt" });
+  });
+
+  test("does not replace newer tool activity when bridge audio drains", async () => {
+    const { starter } = scriptedStartVoiceTurn({
+      frontDoor: ["[1] ", "Let me think about that."],
+      holdEscalated: true,
+    });
+    let releaseBridge = (): void => {};
+    const bridgePending = new Promise<void>((resolve) => {
+      releaseBridge = resolve;
+    });
+    const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
+      options.onAudioChunk({
+        type: "tts_audio",
+        contentType: "audio/pcm",
+        sampleRate: 24_000,
+        dataBase64: "AA==",
+      });
+      await bridgePending;
+      return {
+        provider: "fish-audio" as const,
+        contentType: "audio/pcm",
+        sampleRate: 24_000,
+        chunks: 1,
+        bytes: 1,
+      };
+    });
+    const liveActivityReporter = new RecordingLiveActivityReporter(
+      "conversation-123",
+    );
+    const { frames, session } = createHarness(starter, {
+      streamTtsAudio,
+      liveActivityReporter,
+    });
+
+    await driveTurn(session);
+    await waitFor(() => starter.mock.calls.length >= 2);
+    starter.mock.calls[1]?.[0]?.callbacks?.tool_use_start?.("web_search", {
+      toolUseId: "tool-1",
+    });
+    await waitFor(() => {
+      const activity = frames
+        .filter((frame) => frame.type === "activity")
+        .at(-1);
+      return activity?.label !== "" && activity?.kind === undefined;
+    });
+
+    const escalationFramesBeforeDrain = frames.filter(
+      (frame) => frame.type === "activity" && frame.kind === "escalation",
+    ).length;
+    const overlayBeforeDrain = frames
+      .filter((frame) => frame.type === "activity")
+      .at(-1);
+    releaseBridge();
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    await liveActivityReporter.waitForDispatches();
+
+    expect(
+      frames.filter(
+        (frame) => frame.type === "activity" && frame.kind === "escalation",
+      ),
+    ).toHaveLength(escalationFramesBeforeDrain);
+    const latestActivity = frames
+      .filter((frame) => frame.type === "activity")
+      .at(-1);
+    expect(latestActivity).toEqual(
+      expect.objectContaining({ label: expect.not.stringMatching(/^$/) }),
+    );
+    expect(latestActivity).not.toHaveProperty("kind");
+    expect(liveActivityReporter.dispatched.at(-1)).toEqual({
+      phase: "thinking",
+      event: "update",
+      detail:
+        overlayBeforeDrain?.type === "activity" ? overlayBeforeDrain.label : "",
+    });
+
+    starter.mock.calls[1]?.[0]?.callbacks?.tool_result?.({
+      toolName: "web_search",
+      toolUseId: "tool-1",
+      resultPreview: "",
+    });
+    await waitFor(() => {
+      const activity = frames
+        .filter((frame) => frame.type === "activity")
+        .at(-1);
+      return activity?.kind === "escalation";
+    });
+
+    await session.handleClientFrame({ type: "interrupt" });
   });
 
   test("no leg is told to refuse setup flows, and the escalated leg is told to run them", async () => {
@@ -462,3 +643,45 @@ describe("live-voice triage-and-escalate routing", () => {
     expect(escalatedAbort).toHaveBeenCalledTimes(1);
   });
 });
+
+describe("live-voice escalation judge", () => {
+  function lateVerdict(escalate: boolean, delayMs = 30): Promise<boolean> {
+    return new Promise((resolve) =>
+      setTimeout(() => resolve(escalate), delayMs),
+    );
+  }
+
+  test("a judge escalation overrules an answer the front door already finished", async () => {
+    const { starter, frontDoorOverrule } = scriptedStartVoiceTurn({
+      frontDoor: ["Yeah okay, ", "I'll do it."],
+      escalated: ["Done, the blurb is in the draft."],
+      judgeVerdict: lateVerdict(true),
+    });
+    const { frames, session } = createHarness(starter);
+
+    await driveTurn(session);
+    await waitFor(() => starter.mock.calls.length >= 2);
+    await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
+
+    expect(starter.mock.calls[1]?.[0]?.routingLeg).toBe("escalated");
+    expect(frontDoorOverrule).toHaveBeenCalledTimes(1);
+    // The overruled answer is never heard; the escalated leg answers.
+    expect(spokenText(frames)).not.toContain("I'll do it.");
+    expect(spokenText(frames)).toContain("Done, the blurb is in the draft.");
+  });
+
+  test("a judge that clears late releases the held answer", async () => {
+    const { starter, frontDoorOverrule } = scriptedStartVoiceTurn({
+      frontDoor: ["Sure, ", "it's Tuesday."],
+      judgeVerdict: lateVerdict(false),
+    });
+    const { frames, session } = createHarness(starter);
+
+    await driveTurn(session);
+    await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
+
+    expect(starter).toHaveBeenCalledTimes(1);
+    expect(frontDoorOverrule).not.toHaveBeenCalled();
+    expect(spokenText(frames)).toBe("Sure, it's Tuesday.");
+  });
+});

package/src/live-voice/__tests__/live-voice-tts-session.test.ts

@@ -10,6 +10,7 @@ import type {
 } from "../../stt/types.js";
 import {
   LiveVoiceSession,
+  type LiveVoiceSessionAudioArchiver,
   type LiveVoiceTtsStreamer,
   type LiveVoiceTurnStarter,
 } from "../live-voice-session.js";
@@ -86,6 +87,7 @@ function createContext(): {
 function createSessionHarness(options: {
   startVoiceTurn: LiveVoiceTurnStarter;
   streamTtsAudio: LiveVoiceTtsStreamer | null;
+  archiveAudio?: LiveVoiceSessionAudioArchiver;
 }) {
   const transcriber = new MockStreamingTranscriber();
   const { context, frames } = createContext();
@@ -93,6 +95,7 @@ function createSessionHarness(options: {
     resolveTranscriber: mock(async () => transcriber),
     startVoiceTurn: options.startVoiceTurn,
     streamTtsAudio: options.streamTtsAudio,
+    archiveAudio: options.archiveAudio,
     createTurnId: () => "live-turn-1",
   });
 
@@ -457,6 +460,104 @@ describe("LiveVoiceSession TTS", () => {
     });
   });
 
+  test("drains paced audio in sample order before tts_done", async () => {
+    let callbacks: VoiceTurnCallbacks | undefined;
+    const { session, frames } = createSessionHarness({
+      startVoiceTurn: async (options) => {
+        callbacks = options.callbacks;
+        return { turnId: "bridge-turn-1", abort: mock() };
+      },
+      streamTtsAudio: async (options) => {
+        for (let index = 0; index < 10; index += 1) {
+          options.onAudioChunk({
+            ...makeTtsChunk(""),
+            dataBase64: Buffer.alloc(4800, index).toString("base64"),
+          });
+        }
+        return makeTtsResult("");
+      },
+    });
+    await startReleasedTurn(session);
+    try {
+      callbacks?.assistant_text_delta?.(makeTextDelta(FIRST_SENTENCE));
+      callbacks?.message_complete?.(makeMessageComplete());
+      const deadline = Date.now() + 2000;
+      while (
+        !frames.some((frame) => frame.type === "tts_done") &&
+        Date.now() < deadline
+      ) {
+        await new Promise((resolve) => setTimeout(resolve, 20));
+      }
+      expect(
+        frames
+          .filter((frame) => frame.type === "tts_audio")
+          .map((frame) => Buffer.from(frame.dataBase64, "base64")[0]),
+      ).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+      expect(frames.at(-1)?.type).toBe("tts_done");
+    } finally {
+      await session.close("client_end");
+    }
+  });
+
+  test.each(["interrupt", "close"])(
+    "archives only sent audio when %s stops a paced burst",
+    async (stop) => {
+      let callbacks: VoiceTurnCallbacks | undefined;
+      const archivedAudio: Buffer[] = [];
+      const { session, frames } = createSessionHarness({
+        archiveAudio: async (input) => {
+          if (input.role === "assistant") {
+            archivedAudio.push(Buffer.from(input.audio.dataBase64, "base64"));
+          }
+          return {
+            type: "warning",
+            warning: { code: "archive_failed", message: "held in test" },
+          };
+        },
+        startVoiceTurn: async (options) => {
+          callbacks = options.callbacks;
+          return { turnId: "bridge-turn-1", abort: mock() };
+        },
+        streamTtsAudio: async (options) => {
+          for (let index = 0; index < 20; index += 1) {
+            options.onAudioChunk({
+              ...makeTtsChunk(""),
+              dataBase64: Buffer.alloc(4800, index).toString("base64"),
+            });
+          }
+          return makeTtsResult("");
+        },
+      });
+      await startReleasedTurn(session);
+      try {
+        callbacks?.assistant_text_delta?.(makeTextDelta(FIRST_SENTENCE));
+        callbacks?.message_complete?.(makeMessageComplete());
+        const audioCount = () =>
+          frames.filter((frame) => frame.type === "tts_audio").length;
+        await waitFor(() => audioCount() >= 5);
+        expect(audioCount()).toBeLessThanOrEqual(7);
+        const countBefore = audioCount();
+        if (stop === "interrupt") {
+          await session.handleClientFrame({ type: "interrupt" });
+        } else {
+          await session.close("client_end");
+        }
+        await new Promise((resolve) => setTimeout(resolve, 150));
+        expect(audioCount()).toBe(countBefore);
+        expect(frames.some((frame) => frame.type === "tts_done")).toBe(false);
+        expect(archivedAudio).toEqual([
+          Buffer.concat(
+            ttsAudioPayloads(frames).map((audio) =>
+              Buffer.from(audio, "base64"),
+            ),
+          ),
+        ]);
+      } finally {
+        await session.close("client_end");
+      }
+    },
+  );
+
   test("interrupt prevents late TTS chunks from reaching the socket", async () => {
     let callbacks: VoiceTurnCallbacks | undefined;
     let ttsOptions: LiveVoiceTtsOptions | undefined;

package/src/live-voice/__tests__/live-voice-tts.test.ts

@@ -206,6 +206,50 @@ describe("streamLiveVoiceTtsAudio", () => {
     });
   });
 
+  test.each([2, 32000])(
+    "frames PCM from provider reads of %i bytes without losing samples",
+    async (readBytes) => {
+      config = makeConfig({ provider: "elevenlabs" });
+      const audio = Buffer.alloc(32000, 7);
+      const frames: LiveVoiceTtsAudioChunk[] = [];
+      _setTtsProviderForTests({
+        id: "elevenlabs",
+        capabilities: { supportsStreaming: true, supportedFormats: ["pcm"] },
+        async synthesize() {
+          throw new Error("streaming required");
+        },
+        async synthesizeStream(_request, onChunk) {
+          for (let offset = 0; offset < audio.length; offset += readBytes) {
+            onChunk(audio.subarray(offset, offset + readBytes));
+            if (offset === 0) {
+              await new Promise((resolve) => setTimeout(resolve, 0));
+              expect(frames.length).toBeGreaterThan(0);
+            }
+          }
+          return { audio, contentType: "audio/pcm" };
+        },
+      });
+      const result = await streamLiveVoiceTtsAudio({
+        config,
+        text: "hello",
+        outputFormat: "pcm",
+        sampleRate: 16000,
+        onAudioChunk: (chunk) => frames.push(chunk),
+      });
+      const buffers = frames.map((frame) =>
+        Buffer.from(frame.dataBase64, "base64"),
+      );
+      expect(Buffer.concat(buffers)).toEqual(audio);
+      expect(
+        buffers.every(
+          (buffer) => buffer.length <= 3200 && buffer.length % 2 === 0,
+        ),
+      ).toBe(true);
+      expect(result.chunks).toBeLessThanOrEqual(11);
+      expect(result.bytes).toBe(audio.length);
+    },
+  );
+
   test("carries a trailing odd byte into the next PCM chunk to keep frames sample-aligned", async () => {
     config = makeConfig({ provider: "elevenlabs" });
     _setTtsProviderForTests({