@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

package/src/live-voice/__tests__/live-voice-archive.test.ts

@@ -96,7 +96,7 @@ describe("live voice audio archive", () => {
     const { message } = await createMessage("user");
     const audio = Buffer.from("user audio bytes");
 
-    const result = archiveLiveVoiceUserUtteranceAudio({
+    const result = await archiveLiveVoiceUserUtteranceAudio({
       messageId: message.id,
       sessionId: "session-123",
       turnId: "turn-abc",
@@ -148,7 +148,7 @@ describe("live voice audio archive", () => {
     const audio = Buffer.from("assistant audio bytes");
     writeFileSync(sourcePath, audio);
 
-    const result = archiveLiveVoiceAssistantResponseAudio({
+    const result = await archiveLiveVoiceAssistantResponseAudio({
       messageId: message.id,
       sessionId: "session-456",
       turnId: "turn-def",
@@ -202,7 +202,7 @@ describe("live voice audio archive", () => {
   test("links user utterance audio to a persisted user message id", async () => {
     const { message } = await createMessage("user");
 
-    const result = linkLiveVoiceUserUtteranceAudioToMessage({
+    const result = await linkLiveVoiceUserUtteranceAudioToMessage({
       messageId: message.id,
       sessionId: "session-user-link",
       turnId: "turn-user-link",
@@ -228,7 +228,7 @@ describe("live voice audio archive", () => {
   test("links assistant response audio when the assistant message id is available", async () => {
     const { message } = await createMessage("assistant");
 
-    const result = linkLiveVoiceAssistantResponseAudioToMessage({
+    const result = await linkLiveVoiceAssistantResponseAudioToMessage({
       messageId: message.id,
       sessionId: "session-assistant-link",
       turnId: "turn-assistant-link",
@@ -254,8 +254,8 @@ describe("live voice audio archive", () => {
     expect(getLiveVoiceArtifacts(message.id)).toEqual([result.artifact]);
   });
 
-  test("returns an unlinked result when the assistant message id is unavailable", () => {
-    const result = linkLiveVoiceAssistantResponseAudioToMessage({
+  test("returns an unlinked result when the assistant message id is unavailable", async () => {
+    const result = await linkLiveVoiceAssistantResponseAudioToMessage({
       messageId: undefined,
       sessionId: "session-assistant-unlinked",
       turnId: "turn-assistant-unlinked",
@@ -283,7 +283,7 @@ describe("live voice audio archive", () => {
   test("is idempotent for the session turn role key", async () => {
     const { message } = await createMessage("user");
 
-    const first = archiveLiveVoiceAudioArtifact({
+    const first = await archiveLiveVoiceAudioArtifact({
       messageId: message.id,
       sessionId: "session-repeat",
       turnId: "turn-repeat",
@@ -295,7 +295,7 @@ describe("live voice audio archive", () => {
         dataBase64: Buffer.from("first audio").toString("base64"),
       },
     });
-    const second = archiveLiveVoiceAudioArtifact({
+    const second = await archiveLiveVoiceAudioArtifact({
       messageId: message.id,
       sessionId: "session-repeat",
       turnId: "turn-repeat",
@@ -320,7 +320,7 @@ describe("live voice audio archive", () => {
       "first audio",
     );
 
-    const assistantResult = archiveLiveVoiceAudioArtifact({
+    const assistantResult = await archiveLiveVoiceAudioArtifact({
       messageId: message.id,
       sessionId: "session-repeat",
       turnId: "turn-repeat",
@@ -337,7 +337,7 @@ describe("live voice audio archive", () => {
 
   test("restores metadata idempotency from the deterministic attachment filename", async () => {
     const { message } = await createMessage("assistant");
-    const first = archiveLiveVoiceAssistantResponseAudio({
+    const first = await archiveLiveVoiceAssistantResponseAudio({
       messageId: message.id,
       sessionId: "session-crash",
       turnId: "turn-crash",
@@ -360,7 +360,7 @@ describe("live voice audio archive", () => {
       message.id,
     );
 
-    const second = archiveLiveVoiceAssistantResponseAudio({
+    const second = await archiveLiveVoiceAssistantResponseAudio({
       messageId: message.id,
       sessionId: "session-crash",
       turnId: "turn-crash",
@@ -408,7 +408,7 @@ describe("live voice audio archive", () => {
       },
     );
 
-    const archived = archiveLiveVoiceUserUtteranceAudio({
+    const archived = await archiveLiveVoiceUserUtteranceAudio({
       messageId: sourceMessage.id,
       sessionId: "session-artifact-link",
       turnId: "turn-artifact-link",
@@ -452,7 +452,7 @@ describe("live voice audio archive", () => {
   test("returns typed warnings for non-fatal archive failures", async () => {
     const { message } = await createMessage("user");
 
-    const missingFile = archiveLiveVoiceUserUtteranceAudio({
+    const missingFile = await archiveLiveVoiceUserUtteranceAudio({
       messageId: message.id,
       sessionId: "session-warning",
       turnId: "turn-warning",
@@ -470,7 +470,7 @@ describe("live voice audio archive", () => {
       },
     });
 
-    const unsupportedMime = archiveLiveVoiceUserUtteranceAudio({
+    const unsupportedMime = await archiveLiveVoiceUserUtteranceAudio({
       messageId: message.id,
       sessionId: "session-warning",
       turnId: "turn-warning-2",
@@ -488,7 +488,7 @@ describe("live voice audio archive", () => {
       },
     });
 
-    const missingMessage = archiveLiveVoiceUserUtteranceAudio({
+    const missingMessage = await archiveLiveVoiceUserUtteranceAudio({
       messageId: "missing-message",
       sessionId: "session-warning",
       turnId: "turn-warning-3",
@@ -509,7 +509,7 @@ describe("live voice audio archive", () => {
   test("keeps archive metadata scoped to allowed live voice fields", async () => {
     const { message } = await createMessage("assistant");
 
-    const result = archiveLiveVoiceAssistantResponseAudio({
+    const result = await archiveLiveVoiceAssistantResponseAudio({
       messageId: message.id,
       sessionId: "session-metadata",
       turnId: "turn-metadata",

package/src/live-voice/__tests__/live-voice-attach-image.test.ts

@@ -0,0 +1,209 @@
+/**
+ * Photos taken mid-call (the voice room's camera).
+ *
+ * The behaviour under test is the parking rule: an `attach_image` frame does
+ * not dispatch anything, it waits for the next turn and rides that turn's own
+ * user message. That is what makes a bare "what's this?" resolve against the
+ * picture instead of producing one turn about the image racing another about
+ * the words.
+ */
+
+import { describe, expect, mock, test } from "bun:test";
+
+import type { VoiceTurnOptions } from "../../calls/voice-session-bridge.js";
+import type {
+  StreamingTranscriber,
+  SttStreamServerEvent,
+} from "../../stt/types.js";
+import {
+  LiveVoiceSession,
+  type LiveVoiceTurnStarter,
+} from "../live-voice-session.js";
+import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
+import {
+  createLiveVoiceServerFrameSequencer,
+  type LiveVoiceClientStartFrame,
+  type LiveVoiceServerFrame,
+  validateLiveVoiceClientFrame,
+} from "../protocol.js";
+
+const START_FRAME = {
+  type: "start",
+  conversationId: "conversation-123",
+  audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
+} as const satisfies LiveVoiceClientStartFrame;
+
+class MockStreamingTranscriber implements StreamingTranscriber {
+  readonly providerId = "deepgram" as const;
+  readonly boundaryId = "daemon-streaming" as const;
+  private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
+
+  async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
+    this.onEvent = onEvent;
+  }
+
+  sendAudio(): void {}
+
+  stop(): void {
+    this.emit({ type: "closed" });
+  }
+
+  emit(event: SttStreamServerEvent): void {
+    this.onEvent?.(event);
+  }
+}
+
+function createSessionHarness() {
+  const sequencer = createLiveVoiceServerFrameSequencer();
+  const frames: LiveVoiceServerFrame[] = [];
+  const context: LiveVoiceSessionFactoryContext = {
+    sessionId: "session-123",
+    startFrame: START_FRAME,
+    sendFrame: mock(async (payload) => {
+      const frame = sequencer.next(payload);
+      frames.push(frame);
+      return frame;
+    }),
+  };
+
+  const transcriber = new MockStreamingTranscriber();
+  const startVoiceTurn: LiveVoiceTurnStarter = mock(
+    async (_options: VoiceTurnOptions) => ({
+      turnId: "bridge-turn-1",
+      abort: mock(),
+    }),
+  );
+
+  const session = new LiveVoiceSession(context, {
+    resolveTranscriber: mock(async () => transcriber),
+    startVoiceTurn,
+    createTurnId: () => "live-turn-1",
+    emitMetrics: false,
+  });
+
+  return { frames, session, startVoiceTurn, transcriber };
+}
+
+/** Drive one utterance to a dispatched assistant turn. */
+async function speakAndRelease(
+  harness: Awaited<ReturnType<typeof createSessionHarness>>,
+  text: string,
+): Promise<void> {
+  harness.transcriber.emit({ type: "final", text });
+  await harness.session.handleClientFrame({ type: "ptt_release" });
+  for (let attempt = 0; attempt < 40; attempt += 1) {
+    if (
+      (harness.startVoiceTurn as ReturnType<typeof mock>).mock.calls.length > 0
+    ) {
+      return;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 5));
+  }
+}
+
+function turnOptionsAt(
+  startVoiceTurn: LiveVoiceTurnStarter,
+  index: number,
+): VoiceTurnOptions | undefined {
+  return (startVoiceTurn as ReturnType<typeof mock>).mock.calls[index]?.[0] as
+    | VoiceTurnOptions
+    | undefined;
+}
+
+describe("live-voice attach_image frame", () => {
+  test("accepts a well-formed frame", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "attach_image",
+      attachmentId: "att-1",
+    });
+    expect(result).toEqual({
+      ok: true,
+      frame: { type: "attach_image", attachmentId: "att-1" },
+    });
+  });
+
+  test("rejects a missing attachmentId", () => {
+    const result = validateLiveVoiceClientFrame({ type: "attach_image" });
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.error.code).toBe("missing_required_field");
+      expect(result.error.field).toBe("attachmentId");
+    }
+  });
+
+  test("rejects an empty attachmentId", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "attach_image",
+      attachmentId: "",
+    });
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.error.code).toBe("invalid_field");
+    }
+  });
+});
+
+describe("live-voice photos taken mid-call", () => {
+  test("a photo does not dispatch a turn of its own", async () => {
+    // The photo becomes its own user message via `persistLiveVoicePhoto`; no
+    // assistant turn runs for it. Dispatching one would answer the picture
+    // while the user is still saying the sentence it belongs to.
+    const harness = createSessionHarness();
+    await harness.session.start();
+
+    await harness.session.handleClientFrame({
+      type: "attach_image",
+      attachmentId: "att-1",
+    });
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(harness.startVoiceTurn).not.toHaveBeenCalled();
+  });
+
+  test("a spoken turn carries no attachments of its own", async () => {
+    // Photos reach the model through conversation history, not through the
+    // turn's options, which is what makes shutter-then-speak and
+    // speak-then-shutter behave identically.
+    const harness = createSessionHarness();
+    await harness.session.start();
+
+    await harness.session.handleClientFrame({
+      type: "attach_image",
+      attachmentId: "att-1",
+    });
+    await speakAndRelease(harness, "what's this");
+
+    const options = turnOptionsAt(harness.startVoiceTurn, 0) as
+      | (VoiceTurnOptions & { attachmentIds?: string[] })
+      | undefined;
+    expect(options?.content).toBe("what's this");
+    expect(options?.attachmentIds).toBeUndefined();
+  });
+
+  test("a photo that cannot be stored is reported to the client", async () => {
+    // `att-missing` resolves to nothing (no attachment row in this harness),
+    // so the persist fails. Silence would leave the user believing the
+    // assistant can see something it never received.
+    const harness = createSessionHarness();
+    await harness.session.start();
+    const before = harness.frames.length;
+
+    await harness.session.handleClientFrame({
+      type: "attach_image",
+      attachmentId: "att-missing",
+    });
+    for (let attempt = 0; attempt < 40; attempt += 1) {
+      if (harness.frames.length > before) {
+        break;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 5));
+    }
+
+    const error = harness.frames
+      .slice(before)
+      .find((frame) => frame.type === "error");
+    expect(error).toMatchObject({ type: "error", recoverable: true });
+    // The session survives it: one failed photo is not a failed call.
+    expect(harness.startVoiceTurn).not.toHaveBeenCalled();
+  });
+});

package/src/live-voice/__tests__/live-voice-connection.test.ts

@@ -163,6 +163,26 @@ describe("createLiveVoiceConnection", () => {
     });
   });
 
+  test("names the rejected frame on an unknown_type error", async () => {
+    // A client that sends more than one optional frame gets the same
+    // `unknown_type` for any of them. Without the name it has to guess which,
+    // and guessing wrong is silent: a photo is dropped with nothing said, or
+    // the voice-room settings quietly stop applying for the session.
+    installFakeManager();
+    const { connection, frames } = createConnection();
+
+    await connection.handleMessage(START_MESSAGE);
+    await connection.handleMessage(
+      JSON.stringify({ type: "attach_image_from_the_future" }),
+    );
+
+    expect(frames.at(-1)).toMatchObject({
+      type: "error",
+      code: "unknown_type",
+      frameType: "attach_image_from_the_future",
+    });
+  });
+
   test("errors on a non-start frame before start", async () => {
     installFakeManager();
     const { connection, frames } = createConnection();

package/src/live-voice/__tests__/live-voice-events.test.ts

@@ -4,7 +4,6 @@ import type {
   VoiceTurnCallbacks,
   VoiceTurnOptions,
 } from "../../calls/voice-session-bridge.js";
-import { VOICE_NO_SETUP_FLOWS_RULE } from "../../calls/voice-session-bridge.js";
 import type {
   StreamingTranscriber,
   SttStreamServerEvent,
@@ -254,14 +253,14 @@ describe("LiveVoiceSession archive and metrics events", () => {
       userMessageInterface: "macos",
       assistantMessageInterface: "macos",
       // Pins the full production control prompt for the FRONT-DOOR leg (the
-      // first leg of every routed turn): the speech-first rule (which now
-      // permits a surface, since the room reveals one by itself) and the
-      // shared no-setup-flows rule (no OAuth/browser flows mid-call). The [-1]
-      // room-minimize teaching is deliberately absent — only the escalated
-      // leg learns it (see live-voice-triage-escalate.test.ts).
+      // first leg of every routed turn): the speech-first rule, and nothing
+      // else. This leg is toolless, so both the screen-reveal and setup-flow
+      // teachings are deliberately absent. It has nothing to show and no
+      // connection to run, and its capability digest already routes anything
+      // needing a tool to the escalated leg (see
+      // live-voice-triage-escalate.test.ts).
       voiceControlPrompt:
-        "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. " +
-        VOICE_NO_SETUP_FLOWS_RULE,
+        "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. ",
     });
     callbacks?.assistant_text_delta?.(makeTextDelta("Hello there."));
     callbacks?.message_complete?.(makeMessageComplete());