@vellumai/assistant 0.11.2 → 0.11.3-staging.2

package/src/live-voice/__tests__/live-voice-connection.test.ts

@@ -163,6 +163,26 @@ describe("createLiveVoiceConnection", () => {
     });
   });
 
+  test("names the rejected frame on an unknown_type error", async () => {
+    // A client that sends more than one optional frame gets the same
+    // `unknown_type` for any of them. Without the name it has to guess which,
+    // and guessing wrong is silent: a photo is dropped with nothing said, or
+    // the voice-room settings quietly stop applying for the session.
+    installFakeManager();
+    const { connection, frames } = createConnection();
+
+    await connection.handleMessage(START_MESSAGE);
+    await connection.handleMessage(
+      JSON.stringify({ type: "attach_image_from_the_future" }),
+    );
+
+    expect(frames.at(-1)).toMatchObject({
+      type: "error",
+      code: "unknown_type",
+      frameType: "attach_image_from_the_future",
+    });
+  });
+
   test("errors on a non-start frame before start", async () => {
     installFakeManager();
     const { connection, frames } = createConnection();

package/src/live-voice/__tests__/live-voice-integration.test.ts

@@ -736,9 +736,9 @@ describe("live-voice audio archiving default (JARVIS-1283)", () => {
 
   test("liveVoice.archiveAudio=true wires the default archiver through the factory", async () => {
     // The real linkers write to the DB; these are role-less inputs, so stub
-    // them (synchronously — the linkers are sync) with a valid archived result
-    // per role. The point under test is the config→default-archiver wiring, not
-    // the DB write, which the injected-archiver tests above already cover.
+    // them with a valid archived result per role. The point under test is the
+    // config→default-archiver wiring, not the DB write, which the
+    // injected-archiver tests above already cover.
     const audioInput = {
       sessionId: "session-123",
       turnId: "live-turn-1",
@@ -751,11 +751,13 @@ describe("live-voice audio archiving default (JARVIS-1283)", () => {
     const userSpy = spyOn(
       liveVoiceArchive,
       "linkLiveVoiceUserUtteranceAudioToMessage",
-    ).mockReturnValue(makeArchiveResult({ ...audioInput, role: "user" }));
+    ).mockResolvedValue(makeArchiveResult({ ...audioInput, role: "user" }));
     const assistantSpy = spyOn(
       liveVoiceArchive,
       "linkLiveVoiceAssistantResponseAudioToMessage",
-    ).mockReturnValue(makeArchiveResult({ ...audioInput, role: "assistant" }));
+    ).mockResolvedValue(
+      makeArchiveResult({ ...audioInput, role: "assistant" }),
+    );
     const originalRaw = loadRawConfig();
     saveRawConfig({ ...originalRaw, liveVoice: { archiveAudio: true } });
 

package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts

@@ -0,0 +1,265 @@
+/**
+ * Live-voice session telemetry: the started/ended funnel rows a session
+ * records, and the attribution carried on its turns.
+ *
+ * The recorder is mocked: the outbox and its consent gate are covered by
+ * `telemetry-events-outbox.test.ts`. What these tests pin is the wiring that
+ * decides whether a row exists at all and what it says, which is where the
+ * measurement can silently go wrong: a session that fails before `ready` still
+ * has to count, a failed session still has to record an end, and a session's
+ * turns have to carry its id or the turn count has nothing to group on.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { VoiceTurnOptions } from "../../calls/voice-session-bridge.js";
+import type {
+  StreamingTranscriber,
+  SttStreamServerEvent,
+} from "../../stt/types.js";
+
+const recordLiveVoiceSessionStarted = mock(() => null);
+const recordLiveVoiceSessionEnded = mock(() => null);
+
+// Spread the real module rather than replacing it: other importers pull
+// `recordActivationEvent` and friends out of it, and a bare replacement makes
+// those imports fail to resolve.
+const onboardingEventsStore =
+  await import("../../onboarding/onboarding-events-store.js");
+mock.module("../../onboarding/onboarding-events-store.js", () => ({
+  ...onboardingEventsStore,
+  recordLiveVoiceSessionStarted,
+  recordLiveVoiceSessionEnded,
+}));
+
+// Types are erased, so they import statically regardless of the mock ordering
+// the values below depend on.
+import type { LiveVoiceCredentialReadiness } from "../live-voice-credential-preflight.js";
+import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
+import type {
+  LiveVoiceClientStartFrame,
+  LiveVoiceServerFrame,
+} from "../protocol.js";
+
+// Values load after `mock.module` so the session picks up the mocked recorder.
+const { LiveVoiceSession } = await import("../live-voice-session.js");
+const { LiveVoiceSessionStartupError } =
+  await import("../live-voice-session-manager.js");
+const { createLiveVoiceServerFrameSequencer } = await import("../protocol.js");
+
+const START_FRAME = {
+  type: "start",
+  conversationId: "conversation-123",
+  audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
+} as const satisfies LiveVoiceClientStartFrame;
+
+const NOT_READY: LiveVoiceCredentialReadiness = {
+  status: "not-ready",
+  missing: [
+    {
+      kind: "tts",
+      providerId: "fish-audio",
+      reason: 'TTS provider "fish-audio" is missing credentials',
+    },
+  ],
+  userMessage: "Live voice is unavailable because it requires an API key.",
+};
+
+class MockStreamingTranscriber implements StreamingTranscriber {
+  readonly providerId = "deepgram" as const;
+  readonly boundaryId = "daemon-streaming" as const;
+  private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
+
+  constructor(private readonly stopEvents: SttStreamServerEvent[] = []) {}
+
+  async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
+    this.onEvent = onEvent;
+  }
+
+  sendAudio(): void {}
+
+  stop(): void {
+    for (const event of this.stopEvents) {
+      this.onEvent?.(event);
+    }
+  }
+}
+
+function createContext(
+  startFrame: LiveVoiceClientStartFrame = START_FRAME,
+): LiveVoiceSessionFactoryContext {
+  const sequencer = createLiveVoiceServerFrameSequencer();
+  const frames: LiveVoiceServerFrame[] = [];
+  return {
+    sessionId: "session-123",
+    startFrame,
+    sendFrame: mock(async (payload) => {
+      const frame = sequencer.next(payload);
+      frames.push(frame);
+      return frame;
+    }),
+  };
+}
+
+function readySession(startFrame?: LiveVoiceClientStartFrame) {
+  return new LiveVoiceSession(createContext(startFrame), {
+    resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
+    resolveCredentialReadiness: mock(
+      async (): Promise<LiveVoiceCredentialReadiness> => ({ status: "ready" }),
+    ),
+  });
+}
+
+describe("live-voice session telemetry", () => {
+  beforeEach(() => {
+    recordLiveVoiceSessionStarted.mockClear();
+    recordLiveVoiceSessionEnded.mockClear();
+  });
+
+  test("records a started row for a session that reaches ready", async () => {
+    const session = readySession();
+
+    await session.start();
+
+    expect(recordLiveVoiceSessionStarted).toHaveBeenCalledWith("session-123");
+
+    await session.close("client_end");
+  });
+
+  test("records a started row even when the preflight rejects the session", async () => {
+    const session = new LiveVoiceSession(createContext(), {
+      resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
+      resolveCredentialReadiness: mock(async () => NOT_READY),
+    });
+
+    await expect(session.start()).rejects.toBeInstanceOf(
+      LiveVoiceSessionStartupError,
+    );
+
+    // The denominator of the failure rate: a session rejected before `ready`
+    // is exactly the one that must not go missing from the started count.
+    expect(recordLiveVoiceSessionStarted).toHaveBeenCalledWith("session-123");
+  });
+
+  test("records the close reason and a completed outcome on a clean end", async () => {
+    const session = readySession();
+    await session.start();
+
+    await session.close("client_end");
+
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledWith({
+      sessionId: "session-123",
+      screen: "ended_client_end",
+      outcome: "completed",
+    });
+  });
+
+  test("a dropped socket ends as completed, not failed", async () => {
+    const session = readySession();
+    await session.start();
+
+    await session.close("websocket_close");
+
+    // The session ran and then stopped; only an error makes it `failed`, and
+    // the reason on `screen` is what distinguishes a drop from a hangup.
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledWith({
+      sessionId: "session-123",
+      screen: "ended_websocket_close",
+      outcome: "completed",
+    });
+  });
+
+  test("a failed session records an ended row carrying the failure code", async () => {
+    const session = new LiveVoiceSession(createContext(), {
+      resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
+      resolveCredentialReadiness: mock(async () => NOT_READY),
+    });
+    await expect(session.start()).rejects.toBeInstanceOf(
+      LiveVoiceSessionStartupError,
+    );
+
+    // What the manager does after a startup failure.
+    await session.close("error");
+
+    // A failed session suppresses the client-facing `metrics` frame; the end
+    // row must not be suppressed with it.
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledWith({
+      sessionId: "session-123",
+      screen: "ended_error:credentials_unavailable",
+      outcome: "failed",
+    });
+  });
+
+  test("records exactly one ended row when close is called twice", async () => {
+    const session = readySession();
+    await session.start();
+
+    await session.close("client_end");
+    await session.close("websocket_close");
+
+    expect(recordLiveVoiceSessionEnded).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("live-voice turn attribution", () => {
+  /** Drives one push-to-talk turn and returns the options the bridge saw. */
+  async function captureTurnOptions(
+    startFrame: LiveVoiceClientStartFrame,
+  ): Promise<VoiceTurnOptions> {
+    let captured: VoiceTurnOptions | undefined;
+    const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
+      captured = options;
+      return { turnId: "bridge-turn-1", abort: mock() };
+    });
+    const session = new LiveVoiceSession(createContext(startFrame), {
+      resolveTranscriber: mock(
+        async () =>
+          new MockStreamingTranscriber([
+            { type: "final", text: "hello" },
+            { type: "closed" },
+          ]),
+      ),
+      startVoiceTurn,
+      emitMetrics: false,
+    });
+
+    await session.start();
+    await session.handleClientFrame({ type: "ptt_release" });
+    for (let attempt = 0; attempt < 40 && !captured; attempt += 1) {
+      await new Promise((resolve) => setTimeout(resolve, 5));
+    }
+    await session.close("client_end");
+
+    if (!captured) {
+      throw new Error("the live-voice session never started a bridge turn");
+    }
+    return captured;
+  }
+
+  test("carries the session id and originating client onto the turn", async () => {
+    const options = await captureTurnOptions({ ...START_FRAME, client: "ios" });
+
+    expect(options.voiceTelemetry).toEqual({
+      sessionId: "session-123",
+      client: "ios",
+    });
+  });
+
+  test("leaves the turn's interface id alone while attributing the client", async () => {
+    const options = await captureTurnOptions({ ...START_FRAME, client: "ios" });
+
+    // The load-bearing half of the attribution fix. `userMessageInterface`
+    // resolves the turn's channel capabilities, where `macos` is what grants a
+    // live-voice turn its dynamic surfaces; reporting the true client here
+    // would strip them from every iOS session. Attribution rides
+    // `voiceTelemetry` precisely so this can stay put.
+    expect(options.userMessageInterface).toBe("macos");
+    expect(options.assistantMessageInterface).toBe("macos");
+  });
+
+  test("omits the client when the start frame declares none", async () => {
+    const options = await captureTurnOptions(START_FRAME);
+
+    expect(options.voiceTelemetry).toEqual({ sessionId: "session-123" });
+  });
+});

package/src/live-voice/__tests__/live-voice-stt.test.ts

@@ -203,7 +203,12 @@ describe("LiveVoiceSession STT", () => {
 
     await session.start();
 
-    expect(resolver).toHaveBeenCalledWith({ sampleRate: 24_000 });
+    // The config default is "multi", so the session carries a language into
+    // the resolver rather than leaving it to be inferred downstream.
+    expect(resolver).toHaveBeenCalledWith({
+      sampleRate: 24_000,
+      language: "multi",
+    });
     expect(transcriber.started).toBe(true);
     expect(frames).toEqual([
       {

package/src/live-voice/__tests__/live-voice-vad.test.ts

@@ -103,9 +103,13 @@ class MockStreamingTranscriber implements StreamingTranscriber {
   readonly received: Buffer[] = [];
   stopped = false;
   private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
+  private flushed = false;
+  // Claimed on the first audio chunk, so the scripted transcript is spent in
+  // the order cycles hear speech (see takeScriptedFinal).
+  private scriptedFinal: string | null = null;
 
   constructor(
-    private readonly stopEvents: SttStreamServerEvent[],
+    private readonly takeScriptedFinal: () => string,
     private readonly holdStopEvents = false,
   ) {}
 
@@ -114,6 +118,7 @@ class MockStreamingTranscriber implements StreamingTranscriber {
   }
 
   sendAudio(chunk: Buffer): void {
+    this.scriptedFinal ??= this.takeScriptedFinal();
     this.received.push(Buffer.from(chunk));
   }
 
@@ -128,19 +133,20 @@ class MockStreamingTranscriber implements StreamingTranscriber {
   }
 
   flushStopEvents(): void {
+    if (this.flushed) {
+      return;
+    }
+    this.flushed = true;
     // A stream that was never sent audio has nothing to transcribe, so a real
-    // provider closes without a final. Scripting one anyway let a cycle that
-    // armed and tore down in the same breath — a client interrupt racing the
-    // post-cancel re-arm — hand the session a transcript it never heard, which
-    // launched a turn nothing would ever finalize and wedged the
+    // provider closes without a final. Scripting one anyway lets a cycle that
+    // arms and tears down in the same breath (a client interrupt racing the
+    // post-cancel re-arm) hand the session a transcript it never heard, which
+    // launches a turn nothing will ever finalize and wedges the
     // one-turn-at-a-time gate for every later utterance.
-    const events =
-      this.received.length > 0
-        ? this.stopEvents
-        : this.stopEvents.filter((event) => event.type !== "final");
-    for (const event of events) {
-      this.onEvent?.(event);
+    if (this.scriptedFinal !== null) {
+      this.onEvent?.({ type: "final", text: this.scriptedFinal });
     }
+    this.onEvent?.({ type: "closed" });
   }
 
   // Provider-initiated event (e.g. an idle-timeout close), no stop() needed.
@@ -192,12 +198,23 @@ function createHarness(options: {
   };
 
   const finals = options.finals ?? ["hello world"];
+  // Scripted finals are handed out in the order cycles first receive audio,
+  // not in the order they arm. Server VAD arms and discards audio-free cycles
+  // on wall-clock timers (the trailing-silence timer, the post-cancel re-arm),
+  // so how many cycles have armed by a given point depends on how fast the
+  // event loop is, while the order the test feeds audio in does not. Keying
+  // off audio keeps `finals` reading as "what the user says, in order".
+  let scriptedFinalIndex = 0;
+  const takeScriptedFinal = (): string => {
+    const text =
+      finals[scriptedFinalIndex] ?? `utterance ${scriptedFinalIndex + 1}`;
+    scriptedFinalIndex += 1;
+    return text;
+  };
   const transcribers: MockStreamingTranscriber[] = [];
   const resolveTranscriber = mock(async () => {
-    const text =
-      finals[transcribers.length] ?? `utterance ${transcribers.length + 1}`;
     const transcriber = new MockStreamingTranscriber(
-      [{ type: "final", text }, { type: "closed" }],
+      takeScriptedFinal,
       options.holdStopEventsFor?.includes(transcribers.length) ?? false,
     );
     transcribers.push(transcriber);
@@ -2929,8 +2946,11 @@ describe("LiveVoiceSession server VAD", () => {
 
   test("an idle transcriber close before speech re-arms capture for the next utterance", async () => {
     const { startVoiceTurn, calls } = makeAutoCompletingTurnStarter(["Hi."]);
+    // The first transcriber closes before it is sent any audio, so it never
+    // claims a scripted final: the one entry belongs to the cycle that speech
+    // actually reaches.
     const { frames, session, transcribers } = createHarness({
-      finals: ["never spoken", "hello after close"],
+      finals: ["hello after close"],
       startVoiceTurn,
     });
 

package/src/live-voice/__tests__/protocol.test.ts

@@ -305,6 +305,58 @@ describe("parseLiveVoiceClientTextFrame", () => {
     expect("turnDetection" in result.frame).toBe(false);
   });
 
+  test("parses the originating client from the start frame", () => {
+    const result = parseLiveVoiceClientTextFrame(
+      JSON.stringify({
+        type: "start",
+        client: "ios",
+        audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+      }),
+    );
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) {
+      return;
+    }
+
+    expect(result.frame).toMatchObject({ type: "start", client: "ios" });
+  });
+
+  test("omits client when absent from the start frame", () => {
+    const result = parseLiveVoiceClientTextFrame(
+      JSON.stringify({
+        type: "start",
+        audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+      }),
+    );
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) {
+      return;
+    }
+
+    expect("client" in result.frame).toBe(false);
+  });
+
+  test("drops an unrecognized client rather than rejecting the session", () => {
+    const result = parseLiveVoiceClientTextFrame(
+      JSON.stringify({
+        type: "start",
+        client: "commodore-64",
+        audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+      }),
+    );
+
+    // The field is an analytics dimension. Failing startup over it would
+    // trade a gap in a chart for a user who cannot talk to their assistant.
+    expect(result.ok).toBe(true);
+    if (!result.ok) {
+      return;
+    }
+
+    expect("client" in result.frame).toBe(false);
+  });
+
   test("returns typed protocol errors for invalid turnDetection values", () => {
     const result = validateLiveVoiceClientFrame({
       type: "start",

package/src/live-voice/__tests__/runtime-websocket-shell.test.ts

@@ -327,9 +327,12 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
       conversationId: "conversation-ready",
     });
     expect(typeof ready.sessionId).toBe("string");
+    // The config default is "multi", so the session carries a language into
+    // the resolver rather than leaving it to be inferred downstream.
     expect(resolveStreamingTranscriberMock).toHaveBeenCalledWith({
       sampleRate: 24_000,
       providerId: "deepgram",
+      language: "multi",
     });
     expect(resolvedTranscribers).toHaveLength(1);
     expect(resolvedTranscribers[0]?.started).toBe(true);

package/src/live-voice/live-voice-archive.ts

@@ -468,9 +468,9 @@ function artifactFromAttachment(input: {
   };
 }
 
-export function archiveLiveVoiceAudioArtifact(
+export async function archiveLiveVoiceAudioArtifact(
   input: ArchiveLiveVoiceAudioInput,
-): LiveVoiceAudioArchiveResult {
+): Promise<LiveVoiceAudioArchiveResult> {
   const validated = validateInput(input);
   if ("type" in validated) {
     return validated;
@@ -529,7 +529,7 @@ export function archiveLiveVoiceAudioArtifact(
     const position = input.position ?? nextAttachmentPosition(input.messageId);
     const stored =
       input.audio.type === "base64"
-        ? attachInlineAttachmentToMessage(
+        ? await attachInlineAttachmentToMessage(
             input.messageId,
             position,
             validated.filename,
@@ -614,15 +614,15 @@ interface LinkLiveVoiceAudioArtifactInput {
   position?: number;
 }
 
-export function archiveLiveVoiceUserUtteranceAudio(
+export async function archiveLiveVoiceUserUtteranceAudio(
   input: ArchiveLiveVoiceRolelessAudioInput,
-): LiveVoiceAudioArchiveResult {
+): Promise<LiveVoiceAudioArchiveResult> {
   return archiveLiveVoiceAudioArtifact({ ...input, role: "user" });
 }
 
-export function archiveLiveVoiceAssistantResponseAudio(
+export async function archiveLiveVoiceAssistantResponseAudio(
   input: ArchiveLiveVoiceRolelessAudioInput,
-): LiveVoiceAudioArchiveResult {
+): Promise<LiveVoiceAudioArchiveResult> {
   return archiveLiveVoiceAudioArtifact({ ...input, role: "assistant" });
 }
 
@@ -630,11 +630,11 @@ function normalizeMessageId(messageId: string | null | undefined): string {
   return messageId?.trim() ?? "";
 }
 
-function linkLiveVoiceAudioToMessage(
+async function linkLiveVoiceAudioToMessage(
   input: LinkLiveVoiceRolelessAudioInput & {
     role: LiveVoiceAudioArchiveRole;
   },
-): LiveVoiceAudioArchiveResult {
+): Promise<LiveVoiceAudioArchiveResult> {
   const messageId = normalizeMessageId(input.messageId);
   if (!messageId) {
     return resultUnlinked(
@@ -651,15 +651,15 @@ function linkLiveVoiceAudioToMessage(
   });
 }
 
-export function linkLiveVoiceUserUtteranceAudioToMessage(
+export async function linkLiveVoiceUserUtteranceAudioToMessage(
   input: LinkLiveVoiceRolelessAudioInput,
-): LiveVoiceAudioArchiveResult {
+): Promise<LiveVoiceAudioArchiveResult> {
   return linkLiveVoiceAudioToMessage({ ...input, role: "user" });
 }
 
-export function linkLiveVoiceAssistantResponseAudioToMessage(
+export async function linkLiveVoiceAssistantResponseAudioToMessage(
   input: LinkLiveVoiceRolelessAudioInput,
-): LiveVoiceAudioArchiveResult {
+): Promise<LiveVoiceAudioArchiveResult> {
   return linkLiveVoiceAudioToMessage({ ...input, role: "assistant" });
 }
 

package/src/live-voice/live-voice-connection.ts

@@ -207,13 +207,19 @@ class LiveVoiceConnectionImpl implements LiveVoiceConnection {
   }
 
   private sendError(
-    error: Pick<LiveVoiceProtocolError, "code" | "message">,
+    error: Pick<LiveVoiceProtocolError, "code" | "message" | "frameType">,
   ): void {
     this.sendFrame({
       type: "error",
       seq: this.lastSeq + 1,
       code: error.code,
       message: error.message,
+      // Which frame was rejected, when the parser knows. Without it an
+      // `unknown_type` is unattributable: a client that sends more than one
+      // optional frame cannot tell which of them an older daemon refused, and
+      // has to guess. Guessing wrong is silent, which for a rejected photo
+      // means the user is told nothing at all.
+      ...(error.frameType ? { frameType: error.frameType } : {}),
     });
   }