@vellumai/assistant 0.12.1 → 0.12.2-staging.1

package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts

@@ -0,0 +1,238 @@
+import { describe, expect, mock, spyOn, test } from "bun:test";
+
+import {
+  createMockProvider,
+  textResponse,
+} from "../../__tests__/helpers/mock-provider.js";
+import { setConfig } from "../../__tests__/helpers/set-config.js";
+import { waitFor } from "../../__tests__/helpers/wait-for.js";
+
+setConfig("memory", { enabled: false });
+
+import { UserMessageEchoEventSchema } from "../../api/events/user-message-echo.js";
+import type { AssistantEventEnvelope } from "../../api/index.js";
+import { Conversation } from "../../daemon/conversation.js";
+import * as conversationMessaging from "../../daemon/conversation-messaging.js";
+import {
+  deleteConversation,
+  setConversation,
+} from "../../daemon/conversation-registry.js";
+import { conversationMessagesSyncTag } from "../../daemon/message-types/sync.js";
+import { uploadAttachment } from "../../persistence/attachments-store.js";
+import * as conversationCrud from "../../persistence/conversation-crud.js";
+import { initializeDb } from "../../persistence/db-init.js";
+import { assistantEventHub } from "../../runtime/assistant-event-hub.js";
+import {
+  _drainFrameReclaimRechecksForTests,
+  _pendingFrameReclaimCountForTests,
+  persistAmbientSightFrame,
+  persistLiveVoicePhoto,
+} from "../live-voice-photo.js";
+import { LiveVoiceSession } from "../live-voice-session.js";
+import { createLiveVoiceServerFrameSequencer } from "../protocol.js";
+
+await initializeDb();
+
+function createHarness() {
+  const row = conversationCrud.createConversation("Camera frame echo");
+  const { provider } = createMockProvider([textResponse("")]);
+  const conversation = new Conversation(
+    row.id,
+    provider,
+    "system prompt",
+    () => {},
+    "/tmp",
+    { maxTokens: 4096 },
+  );
+  conversation.setTrustContext({
+    trustClass: "guardian",
+    sourceChannel: "vellum",
+  });
+  setConversation(row.id, conversation);
+
+  const sequencer = createLiveVoiceServerFrameSequencer();
+  const startVoiceTurn = mock(async () => ({
+    turnId: "turn-123",
+    abort: mock(),
+  }));
+  const session = new LiveVoiceSession(
+    {
+      sessionId: "session-123",
+      startFrame: {
+        type: "start",
+        conversationId: row.id,
+        audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
+      },
+      sendFrame: async (payload) => sequencer.next(payload),
+    },
+    {
+      resolveTranscriber: async () => ({
+        providerId: "deepgram",
+        boundaryId: "daemon-streaming",
+        async start() {},
+        sendAudio() {},
+        stop() {},
+      }),
+      startVoiceTurn,
+      emitMetrics: false,
+    },
+  );
+  const published: {
+    event: AssistantEventEnvelope;
+    persistedIds: string[];
+    persistedSeq: number | null;
+  }[] = [];
+  const subscription = assistantEventHub.subscribe({
+    type: "process",
+    filter: { conversationId: row.id },
+    callback: (event) => {
+      if (
+        (event.message.type === "user_message_echo" &&
+          event.message.conversationId === row.id) ||
+        (event.message.type === "sync_changed" &&
+          event.message.tags.includes(conversationMessagesSyncTag(row.id)))
+      ) {
+        published.push({
+          event,
+          persistedIds: conversationCrud.getMessages(row.id).map((m) => m.id),
+          persistedSeq: conversationCrud.getConversationPersistedSeq(row.id),
+        });
+      }
+    },
+  });
+
+  return {
+    conversationId: row.id,
+    session,
+    startVoiceTurn,
+    published,
+    async dispose() {
+      subscription.dispose();
+      await session.close("client_end");
+      deleteConversation(row.id);
+      conversation.dispose();
+    },
+  };
+}
+
+async function uploadFrame(): Promise<string> {
+  const attachment = await uploadAttachment(
+    "frame.png",
+    "image/png",
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk",
+  );
+  return attachment.id;
+}
+
+async function expectAnnouncement(
+  harness: ReturnType<typeof createHarness>,
+  cameraFrame: boolean,
+): Promise<void> {
+  await waitFor(() => harness.published.length >= 2);
+  expect(harness.published.map(({ event }) => event.message.type)).toEqual([
+    "user_message_echo",
+    "sync_changed",
+  ]);
+  const [echo, sync] = harness.published;
+  const message = UserMessageEchoEventSchema.parse(echo.event.message);
+  const rows = conversationCrud.getMessages(harness.conversationId);
+  expect(rows).toHaveLength(1);
+  expect(message.messageId).toBe(rows[0].id);
+  expect(echo.persistedIds).toEqual([rows[0].id]);
+  expect(echo.event.seq).toBeNumber();
+  expect(sync.persistedSeq).toBe(echo.event.seq!);
+  expect(message.text).toBe(cameraFrame ? "(camera frame)" : "here's a photo:");
+  if (cameraFrame) {
+    expect(message.cameraFrame).toBe(true);
+  } else {
+    expect(echo.event.message).not.toHaveProperty("cameraFrame");
+  }
+  expect(harness.startVoiceTurn).not.toHaveBeenCalled();
+}
+
+describe("standalone image echoes", () => {
+  test.each(["sight_frame", "attach_image"] as const)(
+    "%s emits one committed row before the history invalidation",
+    async (type) => {
+      const harness = createHarness();
+      try {
+        await harness.session.start();
+        const attachmentId = await uploadFrame();
+        await harness.session.handleClientFrame({ type, attachmentId });
+        await expectAnnouncement(harness, type === "sight_frame");
+      } finally {
+        await harness.dispose();
+      }
+    },
+  );
+
+  test.each([
+    { kind: "sight_frame", deferred: false },
+    { kind: "photo", deferred: false },
+    { kind: "sight_frame", deferred: true },
+  ] as const)(
+    "$kind preserves its echo marker when a committed write throws (deferred: $deferred)",
+    async ({ kind, deferred }) => {
+      const harness = createHarness();
+      const realPersist = conversationMessaging.persistQueuedMessageBody;
+      const realGetMessage = conversationCrud.getMessageById;
+      let committed = false;
+      const persist = spyOn(
+        conversationMessaging,
+        "persistQueuedMessageBody",
+      ).mockImplementation(async (...args) => {
+        await realPersist(...args);
+        committed = true;
+        throw new Error("Post-insert write failed");
+      });
+      const read = spyOn(conversationCrud, "getMessageById").mockImplementation(
+        (...args) => {
+          if (deferred && committed) {
+            throw new Error("Store temporarily unreadable");
+          }
+          return realGetMessage(...args);
+        },
+      );
+      try {
+        const attachmentId = await uploadFrame();
+        const result =
+          kind === "sight_frame"
+            ? await persistAmbientSightFrame(
+                harness.conversationId,
+                attachmentId,
+                "voice",
+              )
+            : await persistLiveVoicePhoto(harness.conversationId, attachmentId);
+        expect(result.ok).toBe(!deferred);
+        if (deferred) {
+          expect(harness.published).toHaveLength(0);
+          expect(_pendingFrameReclaimCountForTests()).toBe(1);
+          _drainFrameReclaimRechecksForTests();
+          expect(_pendingFrameReclaimCountForTests()).toBe(1);
+          read.mockRestore();
+          _drainFrameReclaimRechecksForTests();
+          _drainFrameReclaimRechecksForTests();
+          expect(_pendingFrameReclaimCountForTests()).toBe(0);
+        }
+        await expectAnnouncement(harness, kind === "sight_frame");
+      } finally {
+        persist.mockRestore();
+        read.mockRestore();
+        _drainFrameReclaimRechecksForTests();
+        await harness.dispose();
+      }
+    },
+  );
+
+  test("the strict echo schema accepts only a true camera marker", () => {
+    const echo = { type: "user_message_echo", text: "(camera frame)" };
+    expect(
+      UserMessageEchoEventSchema.safeParse({ ...echo, cameraFrame: false })
+        .success,
+    ).toBe(false);
+    expect(
+      UserMessageEchoEventSchema.safeParse({ ...echo, unexpected: true })
+        .success,
+    ).toBe(false);
+  });
+});

package/src/live-voice/__tests__/live-voice-vad.test.ts

@@ -12,7 +12,7 @@ import {
   loadRawConfig,
   saveRawConfig,
 } from "../../config/loader.js";
-import type { LiveVoiceFrontModelConfig } from "../../config/schemas/live-voice.js";
+import type { VoiceFrontModelConfig } from "../../config/schemas/voice.js";
 import type {
   StreamingTranscriber,
   SttStreamServerEvent,
@@ -183,7 +183,7 @@ function createHarness(options: {
   echoBargeInMargin?: number;
   echoEmaHalfLifeMs?: number;
   echoDrainSlackMs?: number;
-  frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
+  frontModelConfig?: Partial<VoiceFrontModelConfig>;
   emitMetrics?: boolean;
   metricsClock?: () => number;
   // Return a promise to hold a frame's transport write open (a backed-up

package/src/live-voice/__tests__/runtime-websocket-shell.test.ts

@@ -92,36 +92,17 @@ mock.module("../live-voice-credential-preflight.js", () => ({
       : realPreflightModule.resolveLiveVoiceCredentialReadiness(),
 }));
 
-import { CURRENT_POLICY_EPOCH } from "../../runtime/auth/policy.js";
-import { mintToken } from "../../runtime/auth/token-service.js";
+import {
+  mintActorToken,
+  mintGatewayToken,
+  requireHttpAuth,
+  upgradeHeaders,
+  waitForClose,
+} from "../../runtime/__tests__/runtime-ws-test-utils.js";
 import { RuntimeHttpServer } from "../../runtime/http-server.js";
 
 type JsonFrame = Record<string, unknown>;
 
-const savedAuthEnv = {
-  DISABLE_HTTP_AUTH: process.env.DISABLE_HTTP_AUTH,
-};
-
-function mintGatewayToken(): string {
-  return mintToken({
-    aud: "vellum-daemon",
-    sub: "svc:gateway:self",
-    scope_profile: "gateway_ingress_v1",
-    policy_epoch: CURRENT_POLICY_EPOCH,
-    ttlSeconds: 3600,
-  });
-}
-
-function mintActorToken(): string {
-  return mintToken({
-    aud: "vellum-daemon",
-    sub: "actor:self:user-123",
-    scope_profile: "actor_client_v1",
-    policy_epoch: CURRENT_POLICY_EPOCH,
-    ttlSeconds: 3600,
-  });
-}
-
 function startFrame(conversationId = "conversation-123"): string {
   return JSON.stringify({
     type: "start",
@@ -161,33 +142,6 @@ async function waitForOpen(ws: WebSocket, timeoutMs = 2000): Promise<void> {
   });
 }
 
-async function waitForClose(ws: WebSocket, timeoutMs = 2000): Promise<void> {
-  if (ws.readyState === WebSocket.CLOSED) {
-    return;
-  }
-  await new Promise<void>((resolve, reject) => {
-    const timer = setTimeout(() => {
-      cleanup();
-      reject(new Error("Timed out waiting for WebSocket close"));
-    }, timeoutMs);
-    const cleanup = () => {
-      clearTimeout(timer);
-      ws.removeEventListener("close", onClose);
-      ws.removeEventListener("error", onError);
-    };
-    const onClose = () => {
-      cleanup();
-      resolve();
-    };
-    const onError = () => {
-      cleanup();
-      reject(new Error("WebSocket close failed"));
-    };
-    ws.addEventListener("close", onClose);
-    ws.addEventListener("error", onError);
-  });
-}
-
 async function waitForJsonFrame(
   ws: WebSocket,
   timeoutMs = 2000,
@@ -241,6 +195,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
   let baseUrl: string;
   let wsBaseUrl: string;
   let clients: WebSocket[];
+  let restoreAuthEnv: () => void;
 
   beforeAll(() => {
     shellMocksActive = true;
@@ -251,7 +206,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
   });
 
   beforeEach(async () => {
-    delete process.env.DISABLE_HTTP_AUTH;
+    restoreAuthEnv = requireHttpAuth();
     resolveStreamingTranscriberImpl = async () => createResolvedTranscriber();
     resolveStreamingTranscriberMock.mockClear();
     resolvedTranscribers.length = 0;
@@ -268,11 +223,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
       closeClient(client);
     }
     await server.stop();
-    if (savedAuthEnv.DISABLE_HTTP_AUTH === undefined) {
-      delete process.env.DISABLE_HTTP_AUTH;
-    } else {
-      process.env.DISABLE_HTTP_AUTH = savedAuthEnv.DISABLE_HTTP_AUTH;
-    }
+    restoreAuthEnv();
   });
 
   function openLiveVoiceClient(token = mintGatewayToken()): WebSocket {
@@ -284,21 +235,14 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
   }
 
   test("rejects unauthorized upgrades before creating a WebSocket", async () => {
-    const baseHeaders = {
-      Upgrade: "websocket",
-      Connection: "Upgrade",
-      "Sec-WebSocket-Key": "dGhlIHNhbXBsZSBub25jZQ==",
-      "Sec-WebSocket-Version": "13",
-    };
-
     const missingToken = await fetch(`${baseUrl}/v1/live-voice`, {
-      headers: baseHeaders,
+      headers: upgradeHeaders,
     });
     expect(missingToken.status).toBe(401);
 
     const actorToken = await fetch(
       `${baseUrl}/v1/live-voice?token=${mintActorToken()}`,
-      { headers: baseHeaders },
+      { headers: upgradeHeaders },
     );
     expect(actorToken.status).toBe(401);
 
@@ -306,7 +250,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
       `${baseUrl}/v1/live-voice?token=${mintGatewayToken()}`,
       {
         headers: {
-          ...baseHeaders,
+          ...upgradeHeaders,
           Origin: "https://external.example.com",
         },
       },

package/src/live-voice/live-voice-photo.ts

@@ -237,6 +237,7 @@ async function enqueueStandaloneImagePersist(
         [ticket.attachmentId],
         ticket.content,
         uuidv7(),
+        kind,
       );
       return { ok: false };
     }
@@ -389,11 +390,18 @@ function reclaimOrDefer(
   attachmentIds: readonly string[],
   content: string,
   messageId: string,
+  kind: "photo" | "sight_frame",
 ): void {
   if (reclaimDroppedFrame(attachmentIds)) {
     return;
   }
-  deferFrameReclaimDecision(conversationId, messageId, attachmentIds, content);
+  deferFrameReclaimDecision(
+    conversationId,
+    messageId,
+    attachmentIds,
+    content,
+    kind,
+  );
 }
 
 /** Conversations with a standalone-image persist still in flight. */
@@ -429,6 +437,8 @@ interface PendingFrameReclaim {
   attachmentIds: readonly string[];
   /** Row text, so a frame found to have landed can still be announced. */
   content: string;
+  /** The echo shape used if a later recheck finds the row. */
+  kind: "photo" | "sight_frame";
   attempts: number;
 }
 
@@ -465,12 +475,14 @@ function deferFrameReclaimDecision(
   messageId: string,
   attachmentIds: readonly string[],
   content: string,
+  kind: "photo" | "sight_frame",
 ): void {
   pendingFrameReclaims.push({
     conversationId,
     messageId,
     attachmentIds,
     content,
+    kind,
     attempts: 0,
   });
   scheduleFrameReclaimRecheck();
@@ -521,6 +533,7 @@ function announceDeferredImage(pending: PendingFrameReclaim): void {
       pending.conversationId,
       pending.content,
       pending.messageId,
+      pending.kind,
     );
   } catch (err) {
     log.warn(
@@ -700,6 +713,7 @@ function persistStandaloneImage(
         uuidv7(),
         [attachmentId],
         persistOptions.content,
+        kind,
       );
     }
     return Promise.resolve({ ok: false });
@@ -715,6 +729,7 @@ function persistStandaloneImage(
         [attachmentId],
         persistOptions.content,
         uuidv7(),
+        kind,
       );
     }
     return Promise.resolve({ ok: false });
@@ -754,7 +769,7 @@ function dropReplacedImage(
     "Standalone image dropped: its conversation was replaced before the write",
   );
   if (kind === "sight_frame") {
-    reclaimOrDefer(conversationId, [attachmentId], content, uuidv7());
+    reclaimOrDefer(conversationId, [attachmentId], content, uuidv7(), kind);
   }
   return { ok: false };
 }
@@ -806,7 +821,7 @@ async function writeStandaloneImage(
         "Standalone image dropped: its conversation was deleted while it waited",
       );
       if (kind === "sight_frame") {
-        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7());
+        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7(), kind);
       }
       return { ok: false };
     }
@@ -831,7 +846,7 @@ async function writeStandaloneImage(
         "Standalone image timed out waiting for the conversation to go idle",
       );
       if (kind === "sight_frame") {
-        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7());
+        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7(), kind);
       }
       return { ok: false };
     }
@@ -882,7 +897,7 @@ async function writeStandaloneImage(
       // of.
       conversation.markHistoryStaleForForeignScope(persistOptions.trustContext);
 
-      announcePersistedImage(conversationId, content, persisted.id);
+      announcePersistedImage(conversationId, content, persisted.id, kind);
 
       return {
         ok: true,
@@ -930,7 +945,7 @@ async function writeStandaloneImage(
       // `ensureActorScopedHistory` reloads instead of reusing what it holds.
       findConversation(conversationId)?.markHistoryStale();
       try {
-        announcePersistedImage(conversationId, content, requestId);
+        announcePersistedImage(conversationId, content, requestId, kind);
       } catch (announceErr) {
         log.warn(
           { err: announceErr, conversationId, messageId: requestId },
@@ -946,6 +961,7 @@ async function writeStandaloneImage(
           [attachmentId, ...strandedClones],
           content,
           requestId,
+          kind,
         );
       } else {
         // The store would not say whether the row landed. The refusal below
@@ -957,6 +973,7 @@ async function writeStandaloneImage(
           requestId,
           [attachmentId, ...strandedClones],
           content,
+          kind,
         );
       }
     }
@@ -974,12 +991,14 @@ function announcePersistedImage(
   conversationId: string,
   text: string,
   messageId: string,
+  kind: "photo" | "sight_frame",
 ): void {
   broadcastMessage({
     type: "user_message_echo",
     text,
     conversationId,
     messageId,
+    ...(kind === "sight_frame" ? { cameraFrame: true as const } : {}),
   });
   recordConversationPersistedSeq(conversationId, getCurrentSeq());
   publishConversationMessagesChanged(conversationId);