@vellumai/assistant 0.11.7 → 0.11.8-staging.1

package/src/live-voice/__tests__/live-voice-sight-frame.test.ts

@@ -0,0 +1,347 @@
+/**
+ * Ambient camera frames kept mid-call (`sight_frame`).
+ *
+ * The behaviour under test is the keep-stream rule: a kept frame persists
+ * immediately as its own tagged user message and dispatches no turn, so the
+ * transcript is the record of what the assistant saw and the model correlates
+ * a frame with speech by adjacency.
+ *
+ * The two neighbouring contracts are pinned by their own suites: a deliberate
+ * snap persists standalone and untagged (`live-voice-attach-image.test.ts`), a
+ * parked frame persists nothing and rides the next turn
+ * (`live-voice-attach-frame.test.ts`).
+ */
+
+import { describe, expect, mock, test } from "bun:test";
+
+import {
+  createMockProvider,
+  textResponse,
+} from "../../__tests__/helpers/mock-provider.js";
+import { setConfig } from "../../__tests__/helpers/set-config.js";
+import { waitFor } from "../../__tests__/helpers/wait-for.js";
+
+setConfig("memory", { enabled: false });
+
+import type { VoiceTurnOptions } from "../../calls/voice-session-bridge.js";
+import { Conversation } from "../../daemon/conversation.js";
+import {
+  deleteConversation,
+  setConversation,
+} from "../../daemon/conversation-registry.js";
+import {
+  getAttachmentsForMessage,
+  uploadAttachment,
+} from "../../persistence/attachments-store.js";
+import {
+  createConversation,
+  getMessages,
+  selectSightFrameCaptureTimes,
+} from "../../persistence/conversation-crud.js";
+import { sightFrameAttachmentIdsFromMetadata } from "../../persistence/conversation-types.js";
+import { initializeDb } from "../../persistence/db-init.js";
+import type {
+  StreamingTranscriber,
+  SttStreamServerEvent,
+} from "../../stt/types.js";
+import {
+  LiveVoiceSession,
+  type LiveVoiceTurnStarter,
+} from "../live-voice-session.js";
+import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
+import {
+  createLiveVoiceServerFrameSequencer,
+  type LiveVoiceServerFrame,
+  validateLiveVoiceClientFrame,
+} from "../protocol.js";
+
+await initializeDb();
+
+const IMAGE_BASE64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk";
+
+class MockStreamingTranscriber implements StreamingTranscriber {
+  readonly providerId = "deepgram" as const;
+  readonly boundaryId = "daemon-streaming" as const;
+  private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
+
+  async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
+    this.onEvent = onEvent;
+  }
+
+  sendAudio(): void {}
+
+  stop(): void {
+    this.onEvent?.({ type: "closed" });
+  }
+}
+
+/** A real attachment row, so the arrival-time existence check has one to find. */
+async function uploadFrame(): Promise<string> {
+  const attachment = await uploadAttachment(
+    "frame.png",
+    "image/png",
+    IMAGE_BASE64,
+  );
+  return attachment.id;
+}
+
+/**
+ * A session bound to a live conversation, so a kept frame has somewhere real
+ * to land. Returns a `dispose` that unregisters it.
+ */
+function createSessionHarness(title: string) {
+  const conversation = createConversation(title);
+  const { provider } = createMockProvider([textResponse("")]);
+  const activeConversation = new Conversation(
+    conversation.id,
+    provider,
+    "system prompt",
+    () => {},
+    "/tmp",
+    { maxTokens: 4096 },
+  );
+  activeConversation.setTrustContext({
+    trustClass: "guardian",
+    sourceChannel: "vellum",
+  });
+  setConversation(conversation.id, activeConversation);
+
+  const sequencer = createLiveVoiceServerFrameSequencer();
+  const frames: LiveVoiceServerFrame[] = [];
+  const context: LiveVoiceSessionFactoryContext = {
+    sessionId: "session-sight-frame",
+    startFrame: {
+      type: "start",
+      conversationId: conversation.id,
+      audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
+    },
+    sendFrame: mock(async (payload) => {
+      const frame = sequencer.next(payload);
+      frames.push(frame);
+      return frame;
+    }),
+  };
+
+  const startVoiceTurn: LiveVoiceTurnStarter = mock(
+    async (_options: VoiceTurnOptions) => ({
+      turnId: "bridge-turn-1",
+      abort: mock(),
+    }),
+  );
+
+  const session = new LiveVoiceSession(context, {
+    resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
+    startVoiceTurn,
+    createTurnId: () => "live-turn-1",
+    emitMetrics: false,
+  });
+
+  return {
+    activeConversation,
+    conversationId: conversation.id,
+    frames,
+    session,
+    startVoiceTurn,
+    dispose: () => {
+      deleteConversation(conversation.id);
+      activeConversation.dispose();
+    },
+  };
+}
+
+describe("live-voice sight_frame frame", () => {
+  test("accepts a well-formed frame", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "sight_frame",
+      attachmentId: "att-1",
+    });
+    expect(result).toEqual({
+      ok: true,
+      frame: { type: "sight_frame", attachmentId: "att-1" },
+    });
+  });
+
+  test("rejects a missing attachmentId, naming the frame", () => {
+    const result = validateLiveVoiceClientFrame({ type: "sight_frame" });
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.error.code).toBe("missing_required_field");
+      expect(result.error.field).toBe("attachmentId");
+      expect(result.error.frameType).toBe("sight_frame");
+    }
+  });
+
+  test("rejects an empty attachmentId, distinctly from a missing one", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "sight_frame",
+      attachmentId: "",
+    });
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.error.code).toBe("invalid_field");
+      expect(result.error.field).toBe("attachmentId");
+      expect(result.error.frameType).toBe("sight_frame");
+    }
+  });
+
+  test("rejects a null attachmentId", () => {
+    // `attach_frame` reads null as an unpark. A keep has nothing staged to
+    // give up, so null is simply malformed here.
+    const result = validateLiveVoiceClientFrame({
+      type: "sight_frame",
+      attachmentId: null,
+    });
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.error.code).toBe("invalid_field");
+      expect(result.error.frameType).toBe("sight_frame");
+    }
+  });
+
+  test("rejects a non-string attachmentId", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "sight_frame",
+      attachmentId: 7,
+    });
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.error.code).toBe("invalid_field");
+      expect(result.error.frameType).toBe("sight_frame");
+    }
+  });
+});
+
+describe("live-voice camera frames kept mid-call", () => {
+  test("a kept frame persists as its own tagged message and runs no turn", async () => {
+    const harness = createSessionHarness("Sight keep stream");
+    try {
+      await harness.session.start();
+      const attachmentId = await uploadFrame();
+
+      await harness.session.handleClientFrame({
+        type: "sight_frame",
+        attachmentId,
+      });
+      await waitFor(() => getMessages(harness.conversationId).length > 0, {
+        message: "Timed out waiting for the kept frame to persist",
+      });
+
+      const rows = getMessages(harness.conversationId);
+      expect(rows).toHaveLength(1);
+      const [row] = rows;
+      expect(row.role).toBe("user");
+      expect(getAttachmentsForMessage(row.id)).toHaveLength(1);
+
+      // The tag names the attachment on the row that carries it, which is what
+      // retention reads to decide what a later turn still sends in full.
+      const metadata = JSON.parse(row.metadata ?? "{}") as Record<
+        string,
+        unknown
+      >;
+      expect(sightFrameAttachmentIdsFromMetadata(metadata)).toEqual([
+        attachmentId,
+      ]);
+      expect(
+        selectSightFrameCaptureTimes(harness.conversationId).has(attachmentId),
+      ).toBe(true);
+
+      // Nothing is dispatched for the frame itself: answering a picture the
+      // user never asked about would talk over whatever they are saying.
+      expect(harness.startVoiceTurn).not.toHaveBeenCalled();
+      expect(
+        harness.frames.filter((frame) => frame.type === "error"),
+      ).toHaveLength(0);
+    } finally {
+      harness.dispose();
+    }
+  });
+
+  test("an unknown attachment id is refused, recoverably and attributably", async () => {
+    const harness = createSessionHarness("Sight keep unknown id");
+    try {
+      await harness.session.start();
+      const before = harness.frames.length;
+
+      await harness.session.handleClientFrame({
+        type: "sight_frame",
+        attachmentId: "att-missing",
+      });
+      await waitFor(() => harness.frames.length > before, {
+        message: "Timed out waiting for the rejected camera frame's error",
+      });
+
+      expect(
+        harness.frames.slice(before).find((frame) => frame.type === "error"),
+      ).toMatchObject({
+        type: "error",
+        frameType: "sight_frame",
+        recoverable: true,
+      });
+      // Nothing landed and the call carries on.
+      expect(getMessages(harness.conversationId)).toHaveLength(0);
+      expect(harness.startVoiceTurn).not.toHaveBeenCalled();
+    } finally {
+      harness.dispose();
+    }
+  });
+
+  test("refuses an unknown id while a turn is running, without waiting it out", async () => {
+    // The id is checked before the persist is scheduled. Behind the persist it
+    // would be answered only once the turn released the lock, long after the
+    // client could still retract the preview it showed.
+    const harness = createSessionHarness("Sight keep unknown id mid-turn");
+    try {
+      await harness.session.start();
+      harness.activeConversation.setProcessing(true);
+      const before = harness.frames.length;
+
+      await harness.session.handleClientFrame({
+        type: "sight_frame",
+        attachmentId: "att-missing",
+      });
+      await waitFor(() => harness.frames.length > before, {
+        message: "Timed out waiting for the mid-turn rejection",
+      });
+
+      expect(
+        harness.frames.slice(before).find((frame) => frame.type === "error"),
+      ).toMatchObject({ type: "error", frameType: "sight_frame" });
+    } finally {
+      harness.activeConversation.setProcessing(false);
+      harness.dispose();
+    }
+  });
+
+  test("keeps persist one message each", async () => {
+    // One message per keep, no batching: the gate's rate floor separates them,
+    // so there is no natural batch to form.
+    const harness = createSessionHarness("Sight keep stream repeated");
+    try {
+      await harness.session.start();
+      const first = await uploadFrame();
+      const second = await uploadFrame();
+
+      await harness.session.handleClientFrame({
+        type: "sight_frame",
+        attachmentId: first,
+      });
+      await waitFor(() => getMessages(harness.conversationId).length === 1, {
+        message: "Timed out waiting for the first kept frame",
+      });
+      await harness.session.handleClientFrame({
+        type: "sight_frame",
+        attachmentId: second,
+      });
+      await waitFor(() => getMessages(harness.conversationId).length === 2, {
+        message: "Timed out waiting for the second kept frame",
+      });
+
+      const captureTimes = selectSightFrameCaptureTimes(harness.conversationId);
+      expect([...captureTimes.keys()].sort()).toEqual([first, second].sort());
+      expect(harness.startVoiceTurn).not.toHaveBeenCalled();
+    } finally {
+      harness.dispose();
+    }
+  });
+});

package/src/live-voice/__tests__/live-voice-vad.test.ts

@@ -177,6 +177,7 @@ function createHarness(options: {
   archiveAudio?: LiveVoiceSessionAudioArchiver;
   turnDetectorConfig?: TurnDetectorConfig;
   speechEnergyThreshold?: number;
+  noiseFloorMargin?: number;
   bargeInMinSpeechMs?: number;
   echoBargeInMargin?: number;
   echoEmaHalfLifeMs?: number;
@@ -257,6 +258,11 @@ function createHarness(options: {
       options.turnDetectorConfig ??
       (options.viaFactory ? undefined : { silenceThresholdMs: 40 }),
     speechEnergyThreshold: options.speechEnergyThreshold,
+    // Like echoBargeInMargin below: a directly-constructed session gets the
+    // fixed gate unless a test asks for the adaptation, so a case about
+    // something else cannot drift once it feeds ten seconds of audio.
+    noiseFloorMargin:
+      options.noiseFloorMargin ?? (options.viaFactory ? undefined : 0),
     bargeInMinSpeechMs: options.bargeInMinSpeechMs,
     echoBargeInMargin:
       options.echoBargeInMargin ?? (options.viaFactory ? undefined : 1),
@@ -3227,11 +3233,114 @@ describe("LiveVoiceSession VAD threshold configuration", () => {
     expect(countType(raisedGate.frames, "speech_started")).toBe(0);
   });
 
+  // One second of steady room tone at 24kHz, i.e. exactly one noise-floor
+  // block. Ten of these fill the estimator's window.
+  const roomSecond = (amplitude: number): Uint8Array =>
+    pcm(amplitude, SAMPLE_RATE);
+
+  async function listenToRoom(
+    session: LiveVoiceSession,
+    amplitude: number,
+    seconds = 10,
+  ): Promise<void> {
+    for (let second = 0; second < seconds; second += 1) {
+      await session.handleBinaryAudio(roomSecond(amplitude));
+    }
+  }
+
+  test("a noisy room raises the gate over a chunk that would be speech in a quiet one", async () => {
+    const { frames, session } = createHarness({ noiseFloorMargin: 3 });
+    await session.start();
+
+    // Ten seconds of a 500-level room: below the 800 gate, so none of it is
+    // speech, but loud enough that 3x it lands above the gate.
+    await listenToRoom(session, 500);
+    await flushAsyncCallbacks();
+    expect(countType(frames, "speech_started")).toBe(0);
+
+    // 1000 clears the fixed 800 gate and would be speech today. Against a
+    // measured floor of 500 the gate is now 1500, so this is still the room.
+    await session.handleBinaryAudio(pcm(1_000));
+    await flushAsyncCallbacks();
+    expect(countType(frames, "speech_started")).toBe(0);
+
+    // Someone actually speaking still gets through.
+    await session.handleBinaryAudio(pcm(1_600));
+    await waitFor(() => countType(frames, "speech_started") === 1);
+  });
+
+  test("the same room and chunk stay speech with the adaptation disabled", async () => {
+    // Control for the case above: nothing but noiseFloorMargin differs, so a
+    // failure here means the room audio moved the gate some other way.
+    const { frames, session } = createHarness({ noiseFloorMargin: 0 });
+    await session.start();
+
+    await listenToRoom(session, 500);
+    await flushAsyncCallbacks();
+    expect(countType(frames, "speech_started")).toBe(0);
+
+    await session.handleBinaryAudio(pcm(1_000));
+    await waitFor(() => countType(frames, "speech_started") === 1);
+  });
+
+  test("a quiet room leaves the configured gate exactly where it was", async () => {
+    // The adaptation is one-directional: it may raise the gate, never lower it.
+    // A room quieter than the constant must not make barge-in easier than the
+    // configured threshold, or the change could invent self-interruption.
+    const { frames, session } = createHarness({ noiseFloorMargin: 3 });
+    await session.start();
+
+    await listenToRoom(session, 20);
+    await flushAsyncCallbacks();
+
+    await session.handleBinaryAudio(pcm(800));
+    await flushAsyncCallbacks();
+    expect(countType(frames, "speech_started")).toBe(0);
+
+    await session.handleBinaryAudio(pcm(801));
+    await waitFor(() => countType(frames, "speech_started") === 1);
+  });
+
+  test("the raised gate is capped at 4x the configured threshold", async () => {
+    // An absurd margin stands in for a pathological room: whatever the floor
+    // says, the gate must stay somewhere a person can still be heard over.
+    const { frames, session } = createHarness({
+      speechEnergyThreshold: 800,
+      noiseFloorMargin: 100,
+    });
+    await session.start();
+
+    await listenToRoom(session, 300);
+    await flushAsyncCallbacks();
+
+    // Uncapped this would be 30_000, past anything 16-bit audio can reach.
+    await session.handleBinaryAudio(pcm(3_100));
+    await flushAsyncCallbacks();
+    expect(countType(frames, "speech_started")).toBe(0);
+
+    await session.handleBinaryAudio(pcm(3_300));
+    await waitFor(() => countType(frames, "speech_started") === 1);
+  });
+
+  test("a partial window has no opinion, so the gate stays configured", async () => {
+    const { frames, session } = createHarness({ noiseFloorMargin: 3 });
+    await session.start();
+
+    // Nine seconds is not yet a window, and a partial window must not report
+    // the loudest thing it has heard as the room.
+    await listenToRoom(session, 500, 9);
+    await flushAsyncCallbacks();
+
+    await session.handleBinaryAudio(pcm(1_000));
+    await waitFor(() => countType(frames, "speech_started") === 1);
+  });
+
   test("with no config set the factory defaults to 800 energy / 1200 ms silence / 30 s max turn / 250 ms barge-in", async () => {
     // The test workspace has no liveVoice config, so the factory reads the
     // schema defaults.
     expect(getConfig().liveVoice.vad).toEqual({
       speechEnergyThreshold: 800,
+      noiseFloorMargin: 3,
       silenceThresholdMs: 1200,
       maxTurnDurationMs: 30_000,
       bargeInMinSpeechMs: 250,