@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts

@@ -0,0 +1,265 @@
+/**
+ * Live-voice session telemetry: the started/ended funnel rows a session
+ * records, and the attribution carried on its turns.
+ *
+ * The recorder is mocked: the outbox and its consent gate are covered by
+ * `telemetry-events-outbox.test.ts`. What these tests pin is the wiring that
+ * decides whether a row exists at all and what it says, which is where the
+ * measurement can silently go wrong: a session that fails before `ready` still
+ * has to count, a failed session still has to record an end, and a session's
+ * turns have to carry its id or the turn count has nothing to group on.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { VoiceTurnOptions } from "../../calls/voice-session-bridge.js";
+import type {
+  StreamingTranscriber,
+  SttStreamServerEvent,
+} from "../../stt/types.js";
+
+const recordLiveVoiceSessionStarted = mock(() => null);
+const recordLiveVoiceSessionEnded = mock(() => null);
+
+// Spread the real module rather than replacing it: other importers pull
+// `recordActivationEvent` and friends out of it, and a bare replacement makes
+// those imports fail to resolve.
+const onboardingEventsStore =
+  await import("../../onboarding/onboarding-events-store.js");
+mock.module("../../onboarding/onboarding-events-store.js", () => ({
+  ...onboardingEventsStore,
+  recordLiveVoiceSessionStarted,
+  recordLiveVoiceSessionEnded,
+}));
+
+// Types are erased, so they import statically regardless of the mock ordering
+// the values below depend on.
+import type { LiveVoiceCredentialReadiness } from "../live-voice-credential-preflight.js";
+import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
+import type {
+  LiveVoiceClientStartFrame,
+  LiveVoiceServerFrame,
+} from "../protocol.js";
+
+// Values load after `mock.module` so the session picks up the mocked recorder.
+const { LiveVoiceSession } = await import("../live-voice-session.js");
+const { LiveVoiceSessionStartupError } =
+  await import("../live-voice-session-manager.js");
+const { createLiveVoiceServerFrameSequencer } = await import("../protocol.js");
+
+const START_FRAME = {
+  type: "start",
+  conversationId: "conversation-123",
+  audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
+} as const satisfies LiveVoiceClientStartFrame;
+
+const NOT_READY: LiveVoiceCredentialReadiness = {
+  status: "not-ready",
+  missing: [
+    {
+      kind: "tts",
+      providerId: "fish-audio",
+      reason: 'TTS provider "fish-audio" is missing credentials',
+    },
+  ],
+  userMessage: "Live voice is unavailable because it requires an API key.",
+};
+
+class MockStreamingTranscriber implements StreamingTranscriber {
+  readonly providerId = "deepgram" as const;
+  readonly boundaryId = "daemon-streaming" as const;
+  private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
+
+  constructor(private readonly stopEvents: SttStreamServerEvent[] = []) {}
+
+  async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
+    this.onEvent = onEvent;
+  }
+
+  sendAudio(): void {}
+
+  stop(): void {
+    for (const event of this.stopEvents) {
+      this.onEvent?.(event);
+    }
+  }
+}
+
+function createContext(
+  startFrame: LiveVoiceClientStartFrame = START_FRAME,
+): LiveVoiceSessionFactoryContext {
+  const sequencer = createLiveVoiceServerFrameSequencer();
+  const frames: LiveVoiceServerFrame[] = [];
+  return {
+    sessionId: "session-123",
+    startFrame,
+    sendFrame: mock(async (payload) => {
+      const frame = sequencer.next(payload);
+      frames.push(frame);
+      return frame;
+    }),
+  };
+}
+
+function readySession(startFrame?: LiveVoiceClientStartFrame) {
+  return new LiveVoiceSession(createContext(startFrame), {
+    resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
+    resolveCredentialReadiness: mock(
+      async (): Promise<LiveVoiceCredentialReadiness> => ({ status: "ready" }),
+    ),
+  });
+}
+
+describe("live-voice session telemetry", () => {
+  beforeEach(() => {
+    recordLiveVoiceSessionStarted.mockClear();
+    recordLiveVoiceSessionEnded.mockClear();
+  });
+
+  test("records a started row for a session that reaches ready", async () => {
+    const session = readySession();
+
+    await session.start();
+
+    expect(recordLiveVoiceSessionStarted).toHaveBeenCalledWith("session-123");
+
+    await session.close("client_end");
+  });
+
+  test("records a started row even when the preflight rejects the session", async () => {
+    const session = new LiveVoiceSession(createContext(), {
+      resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
+      resolveCredentialReadiness: mock(async () => NOT_READY),
+    });
+
+    await expect(session.start()).rejects.toBeInstanceOf(
+      LiveVoiceSessionStartupError,
+    );
+
+    // The denominator of the failure rate: a session rejected before `ready`
+    // is exactly the one that must not go missing from the started count.
+    expect(recordLiveVoiceSessionStarted).toHaveBeenCalledWith("session-123");
+  });
+
+  test("records the close reason and a completed outcome on a clean end", async () => {
+    const session = readySession();
+    await session.start();
+
+    await session.close("client_end");
+
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledWith({
+      sessionId: "session-123",
+      screen: "ended_client_end",
+      outcome: "completed",
+    });
+  });
+
+  test("a dropped socket ends as completed, not failed", async () => {
+    const session = readySession();
+    await session.start();
+
+    await session.close("websocket_close");
+
+    // The session ran and then stopped; only an error makes it `failed`, and
+    // the reason on `screen` is what distinguishes a drop from a hangup.
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledWith({
+      sessionId: "session-123",
+      screen: "ended_websocket_close",
+      outcome: "completed",
+    });
+  });
+
+  test("a failed session records an ended row carrying the failure code", async () => {
+    const session = new LiveVoiceSession(createContext(), {
+      resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
+      resolveCredentialReadiness: mock(async () => NOT_READY),
+    });
+    await expect(session.start()).rejects.toBeInstanceOf(
+      LiveVoiceSessionStartupError,
+    );
+
+    // What the manager does after a startup failure.
+    await session.close("error");
+
+    // A failed session suppresses the client-facing `metrics` frame; the end
+    // row must not be suppressed with it.
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledWith({
+      sessionId: "session-123",
+      screen: "ended_error:credentials_unavailable",
+      outcome: "failed",
+    });
+  });
+
+  test("records exactly one ended row when close is called twice", async () => {
+    const session = readySession();
+    await session.start();
+
+    await session.close("client_end");
+    await session.close("websocket_close");
+
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("live-voice turn attribution", () => {
+  /** Drives one push-to-talk turn and returns the options the bridge saw. */
+  async function captureTurnOptions(
+    startFrame: LiveVoiceClientStartFrame,
+  ): Promise<VoiceTurnOptions> {
+    let captured: VoiceTurnOptions | undefined;
+    const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
+      captured = options;
+      return { turnId: "bridge-turn-1", abort: mock() };
+    });
+    const session = new LiveVoiceSession(createContext(startFrame), {
+      resolveTranscriber: mock(
+        async () =>
+          new MockStreamingTranscriber([
+            { type: "final", text: "hello" },
+            { type: "closed" },
+          ]),
+      ),
+      startVoiceTurn,
+      emitMetrics: false,
+    });
+
+    await session.start();
+    await session.handleClientFrame({ type: "ptt_release" });
+    for (let attempt = 0; attempt < 40 && !captured; attempt += 1) {
+      await new Promise((resolve) => setTimeout(resolve, 5));
+    }
+    await session.close("client_end");
+
+    if (!captured) {
+      throw new Error("the live-voice session never started a bridge turn");
+    }
+    return captured;
+  }
+
+  test("carries the session id and originating client onto the turn", async () => {
+    const options = await captureTurnOptions({ ...START_FRAME, client: "ios" });
+
+    expect(options.voiceTelemetry).toEqual({
+      sessionId: "session-123",
+      client: "ios",
+    });
+  });
+
+  test("leaves the turn's interface id alone while attributing the client", async () => {
+    const options = await captureTurnOptions({ ...START_FRAME, client: "ios" });
+
+    // The load-bearing half of the attribution fix. `userMessageInterface`
+    // resolves the turn's channel capabilities, where `macos` is what grants a
+    // live-voice turn its dynamic surfaces; reporting the true client here
+    // would strip them from every iOS session. Attribution rides
+    // `voiceTelemetry` precisely so this can stay put.
+    expect(options.userMessageInterface).toBe("macos");
+    expect(options.assistantMessageInterface).toBe("macos");
+  });
+
+  test("omits the client when the start frame declares none", async () => {
+    const options = await captureTurnOptions(START_FRAME);
+
+    expect(options.voiceTelemetry).toEqual({ sessionId: "session-123" });
+  });
+});

package/src/live-voice/__tests__/live-voice-stt.test.ts

@@ -12,6 +12,7 @@ import {
   type LiveVoiceStreamingTranscriberResolver,
 } from "../live-voice-session.js";
 import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
+import type { LiveVoiceTtsOptions } from "../live-voice-tts.js";
 import {
   createLiveVoiceServerFrameSequencer,
   type LiveVoiceClientStartFrame,
@@ -29,7 +30,9 @@ const START_FRAME = {
 } as const satisfies LiveVoiceClientStartFrame;
 
 class MockStreamingTranscriber implements StreamingTranscriber {
-  readonly providerId = "deepgram" as const;
+  // Configurable: the session gates the language-pin fallback on the DIALED
+  // transcriber's providerId (see turnLanguageFor).
+  readonly providerId: StreamingTranscriber["providerId"];
   readonly boundaryId = "daemon-streaming" as const;
   readonly audioChunks: Buffer[] = [];
   readonly mimeTypes: string[] = [];
@@ -37,6 +40,10 @@ class MockStreamingTranscriber implements StreamingTranscriber {
   stopped = false;
   private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
 
+  constructor(providerId: StreamingTranscriber["providerId"] = "deepgram") {
+    this.providerId = providerId;
+  }
+
   async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
     this.started = true;
     this.onEvent = onEvent;
@@ -135,6 +142,39 @@ function completingVoiceTurnStarter() {
   });
 }
 
+// A starter whose leg streams one spoken sentence so the turn reaches TTS.
+function speakingVoiceTurnStarter(reply = "Okay.") {
+  return mock(async (options: VoiceTurnOptions) => {
+    options.callbacks?.assistant_text_delta?.({
+      type: "assistant_text_delta",
+      text: reply,
+      conversationId: options.conversationId,
+    });
+    options.callbacks?.message_complete?.({
+      type: "message_complete",
+      conversationId: options.conversationId,
+      messageId: "assistant-message-123",
+    });
+    return { turnId: "bridge-turn-1", abort: mock() };
+  });
+}
+
+// Records the exact options object of every TTS synthesis request.
+function recordingTtsStreamer() {
+  const ttsCalls: LiveVoiceTtsOptions[] = [];
+  const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
+    ttsCalls.push(options);
+    return {
+      provider: "fish-audio" as const,
+      contentType: "audio/pcm",
+      sampleRate: 24_000,
+      chunks: 0,
+      bytes: 0,
+    };
+  });
+  return { streamTtsAudio, ttsCalls };
+}
+
 function createContext(overrides: Partial<LiveVoiceClientStartFrame> = {}): {
   context: LiveVoiceSessionFactoryContext;
   frames: LiveVoiceServerFrame[];
@@ -203,7 +243,12 @@ describe("LiveVoiceSession STT", () => {
 
     await session.start();
 
-    expect(resolver).toHaveBeenCalledWith({ sampleRate: 24_000 });
+    // The config default is "multi", so the session carries a language into
+    // the resolver rather than leaving it to be inferred downstream.
+    expect(resolver).toHaveBeenCalledWith({
+      sampleRate: 24_000,
+      language: "multi",
+    });
     expect(transcriber.started).toBe(true);
     expect(frames).toEqual([
       {
@@ -216,6 +261,20 @@ describe("LiveVoiceSession STT", () => {
     ]);
   });
 
+  test("ignores model-integrated turn-detection events", async () => {
+    const { frames, session, transcriber } = createSessionWithTranscriber();
+
+    await session.start();
+    transcriber.emit({ type: "turn-start" });
+    transcriber.emit({ type: "eager-turn-end", text: "eager" });
+    transcriber.emit({ type: "turn-resumed" });
+    transcriber.emit({ type: "turn-end", text: "committed", confidence: 0.9 });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(frames.map((frame) => frame.type)).toEqual(["ready"]);
+    expect(session.finalTranscriptText).toBe("");
+  });
+
   test("marks transient transcriber errors recoverable while the session continues", async () => {
     const { frames, session, transcriber } = createSessionWithTranscriber();
 
@@ -1332,6 +1391,269 @@ describe("LiveVoiceSession STT", () => {
     ]);
   });
 
+  test("finals tagged with a detected language carry it to TTS and the control prompt", async () => {
+    const transcriber = new MockStreamingTranscriber();
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    await session.start();
+    await session.handleBinaryAudio(new Uint8Array([1]));
+    transcriber.emit({ type: "final", text: "नमस्ते", languages: ["hi"] });
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => ttsCalls.length >= 1);
+
+    expect(ttsCalls[0]?.language).toBe("hi");
+    expect(startVoiceTurn.mock.calls[0]?.[0]?.voiceControlPrompt).toContain(
+      'speaking the language with code "hi"',
+    );
+  });
+
+  test("a tagged partial supplies the turn language when finals carry no tags", async () => {
+    // Speculative turns dispatch from partials before the first tagged
+    // final, so the latest tagged partial must resolve the language.
+    const transcriber = new MockStreamingTranscriber();
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    await session.start();
+    await session.handleBinaryAudio(new Uint8Array([1]));
+    transcriber.emit({ type: "partial", text: "hola", languages: ["es"] });
+    transcriber.emit({ type: "final", text: "hola amigo" });
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => ttsCalls.length >= 1);
+
+    expect(ttsCalls[0]?.language).toBe("es");
+  });
+
+  test("empty finals carrying language tags do not vote", async () => {
+    // Silence frames can carry container-level tags describing no emitted
+    // words; they must not outvote the language of real speech.
+    const transcriber = new MockStreamingTranscriber();
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    await session.start();
+    await session.handleBinaryAudio(new Uint8Array([1]));
+    transcriber.emit({ type: "final", text: "", languages: ["es"] });
+    transcriber.emit({ type: "final", text: "", languages: ["es"] });
+    transcriber.emit({ type: "final", text: "hello there", languages: ["en"] });
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => ttsCalls.length >= 1);
+
+    expect(ttsCalls[0]?.language).toBe("en");
+  });
+
+  test("only each final's dominant language votes", async () => {
+    // `languages` is dominance-ranked per event: a Spanish-dominant segment
+    // tagged ["es", "en"] plus a short English segment tagged ["en"] must
+    // resolve to Spanish (one vote each, tie broken by first appearance),
+    // not English by counting the secondary tag as a full vote.
+    const transcriber = new MockStreamingTranscriber();
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    await session.start();
+    await session.handleBinaryAudio(new Uint8Array([1]));
+    transcriber.emit({
+      type: "final",
+      text: "hola amigo como estas hoy",
+      languages: ["es", "en"],
+    });
+    transcriber.emit({ type: "final", text: "ok", languages: ["en"] });
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => ttsCalls.length >= 1);
+
+    expect(ttsCalls[0]?.language).toBe("es");
+  });
+
+  test("a tagged final outranks an earlier tagged partial", async () => {
+    const transcriber = new MockStreamingTranscriber();
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    await session.start();
+    await session.handleBinaryAudio(new Uint8Array([1]));
+    transcriber.emit({ type: "partial", text: "hola", languages: ["es"] });
+    transcriber.emit({ type: "final", text: "नमस्ते", languages: ["hi"] });
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => ttsCalls.length >= 1);
+
+    expect(ttsCalls[0]?.language).toBe("hi");
+  });
+
+  test("a monolingual services.stt.language pin is the turn language when finals carry no tags", async () => {
+    const originalRaw = loadRawConfig();
+    const rawServices = (originalRaw.services ?? {}) as Record<string, unknown>;
+    saveRawConfig({
+      ...originalRaw,
+      services: {
+        ...rawServices,
+        stt: {
+          ...((rawServices.stt ?? {}) as Record<string, unknown>),
+          language: "ja",
+        },
+      },
+    });
+    const transcriber = new MockStreamingTranscriber();
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    try {
+      await session.start();
+      await session.handleBinaryAudio(new Uint8Array([1]));
+      await session.handleClientFrame({ type: "ptt_release" });
+      await waitFor(() => ttsCalls.length >= 1);
+
+      expect(ttsCalls[0]?.language).toBe("ja");
+      expect(startVoiceTurn.mock.calls[0]?.[0]?.voiceControlPrompt).toContain(
+        'speaking the language with code "ja"',
+      );
+    } finally {
+      await session.close("websocket_close");
+      saveRawConfig(originalRaw);
+    }
+  });
+
+  test("a pin on an auto-detecting provider does not become the turn language", async () => {
+    // google-gemini and openai-whisper ignore services.stt.language, so a
+    // stale persisted pin must not force every turn into that language.
+    const originalRaw = loadRawConfig();
+    const rawServices = (originalRaw.services ?? {}) as Record<string, unknown>;
+    saveRawConfig({
+      ...originalRaw,
+      services: {
+        ...rawServices,
+        stt: {
+          ...((rawServices.stt ?? {}) as Record<string, unknown>),
+          provider: "google-gemini",
+          language: "es",
+        },
+      },
+    });
+    const transcriber = new MockStreamingTranscriber("google-gemini");
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    try {
+      await session.start();
+      await session.handleBinaryAudio(new Uint8Array([1]));
+      await session.handleClientFrame({ type: "ptt_release" });
+      await waitFor(() => ttsCalls.length >= 1);
+
+      expect(ttsCalls[0] && "language" in ttsCalls[0]).toBe(false);
+    } finally {
+      await session.close("websocket_close");
+      saveRawConfig(originalRaw);
+    }
+  });
+
+  test("the pin gate follows the dialed provider, not the configured one", async () => {
+    // A BYOK gemini config without credentials silently dials the managed
+    // vellum transcriber, which DOES honor the pin; the gate must follow
+    // what was dialed, not what was configured.
+    const originalRaw = loadRawConfig();
+    const rawServices = (originalRaw.services ?? {}) as Record<string, unknown>;
+    saveRawConfig({
+      ...originalRaw,
+      services: {
+        ...rawServices,
+        stt: {
+          ...((rawServices.stt ?? {}) as Record<string, unknown>),
+          provider: "google-gemini",
+          language: "es",
+        },
+      },
+    });
+    const transcriber = new MockStreamingTranscriber("vellum");
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    try {
+      await session.start();
+      await session.handleBinaryAudio(new Uint8Array([1]));
+      await session.handleClientFrame({ type: "ptt_release" });
+      await waitFor(() => ttsCalls.length >= 1);
+
+      expect(ttsCalls[0]?.language).toBe("es");
+    } finally {
+      await session.close("websocket_close");
+      saveRawConfig(originalRaw);
+    }
+  });
+
+  test('the default "multi" with tag-less finals leaves every language-aware path untouched', async () => {
+    const transcriber = new MockStreamingTranscriber();
+    const startVoiceTurn = speakingVoiceTurnStarter();
+    const { streamTtsAudio, ttsCalls } = recordingTtsStreamer();
+    const { context } = createContext();
+    const session = new LiveVoiceSession(context, {
+      resolveTranscriber: mock(async () => transcriber),
+      startVoiceTurn,
+      streamTtsAudio,
+    });
+
+    await session.start();
+    await session.handleBinaryAudio(new Uint8Array([1]));
+    await session.handleClientFrame({ type: "ptt_release" });
+    await waitFor(() => ttsCalls.length >= 1);
+
+    // With the language unknown the TTS request carries no language key at
+    // all and the control prompt has no per-turn language note: byte-for-byte
+    // the language-blind behavior.
+    expect("language" in ttsCalls[0]!).toBe(false);
+    expect(startVoiceTurn.mock.calls[0]?.[0]?.voiceControlPrompt).not.toContain(
+      "language with code",
+    );
+  });
+
   test("uses the production streaming transcriber resolver by default", () => {
     const source = readFileSync(
       new URL("../live-voice-session.ts", import.meta.url),