@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0

package/src/live-voice/__tests__/protocol.test.ts

@@ -698,6 +698,53 @@ describe("parseLiveVoiceClientTextFrame", () => {
     });
   });
 
+  test("keeps known session controls and drops unknown ones", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "start",
+      sessionControls: [
+        "mute",
+        "look_stop",
+        "look_camera",
+        "end",
+        "mute",
+        7,
+        "fly",
+      ],
+      audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+    });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) {
+      return;
+    }
+    // A newer client's controls cost it nothing on an older daemon.
+    expect(result.frame).toMatchObject({
+      type: "start",
+      sessionControls: ["end", "mute", "look_camera", "look_stop"],
+    });
+  });
+
+  test.each([
+    ["absent", {}],
+    ["not an array", { sessionControls: "end" }],
+    ["all unknown", { sessionControls: ["fly"] }],
+  ])(
+    "omits sessionControls from the start frame when %s",
+    (_label, extra: Record<string, unknown>) => {
+      const result = validateLiveVoiceClientFrame({
+        type: "start",
+        ...extra,
+        audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+      });
+
+      expect(result.ok).toBe(true);
+      if (!result.ok) {
+        return;
+      }
+      expect("sessionControls" in result.frame).toBe(false);
+    },
+  );
+
   test("parses the textInput capability on the start frame", () => {
     const result = validateLiveVoiceClientFrame({
       type: "start",
@@ -753,6 +800,54 @@ describe("parseLiveVoiceClientTextFrame", () => {
     });
   });
 
+  test("parses the lookFrames capability on the start frame", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "start",
+      lookFrames: true,
+      audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+    });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) {
+      return;
+    }
+    expect(result.frame).toMatchObject({ type: "start", lookFrames: true });
+  });
+
+  test("omits lookFrames from the start frame when false", () => {
+    // False and absent mean the same thing: no look frame is coming, so the
+    // session must not wait for one.
+    const result = validateLiveVoiceClientFrame({
+      type: "start",
+      lookFrames: false,
+      audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+    });
+
+    expect(result.ok).toBe(true);
+    if (!result.ok) {
+      return;
+    }
+    expect("lookFrames" in result.frame).toBe(false);
+  });
+
+  test("returns a typed protocol error for a non-boolean lookFrames", () => {
+    const result = validateLiveVoiceClientFrame({
+      type: "start",
+      lookFrames: 1,
+      audio: { mimeType: "audio/pcm", sampleRate: 24000, channels: 1 },
+    });
+
+    expect(result.ok).toBe(false);
+    if (result.ok) {
+      return;
+    }
+    expect(result.error).toMatchObject({
+      code: "invalid_field",
+      field: "lookFrames",
+      frameType: "start",
+    });
+  });
+
   test("returns typed protocol errors for missing audio configuration fields", () => {
     const result = validateLiveVoiceClientFrame({
       type: "start",

package/src/live-voice/__tests__/runtime-websocket-shell.test.ts

@@ -92,36 +92,17 @@ mock.module("../live-voice-credential-preflight.js", () => ({
       : realPreflightModule.resolveLiveVoiceCredentialReadiness(),
 }));
 
-import { CURRENT_POLICY_EPOCH } from "../../runtime/auth/policy.js";
-import { mintToken } from "../../runtime/auth/token-service.js";
+import {
+  mintActorToken,
+  mintGatewayToken,
+  requireHttpAuth,
+  upgradeHeaders,
+  waitForClose,
+} from "../../runtime/__tests__/runtime-ws-test-utils.js";
 import { RuntimeHttpServer } from "../../runtime/http-server.js";
 
 type JsonFrame = Record<string, unknown>;
 
-const savedAuthEnv = {
-  DISABLE_HTTP_AUTH: process.env.DISABLE_HTTP_AUTH,
-};
-
-function mintGatewayToken(): string {
-  return mintToken({
-    aud: "vellum-daemon",
-    sub: "svc:gateway:self",
-    scope_profile: "gateway_ingress_v1",
-    policy_epoch: CURRENT_POLICY_EPOCH,
-    ttlSeconds: 3600,
-  });
-}
-
-function mintActorToken(): string {
-  return mintToken({
-    aud: "vellum-daemon",
-    sub: "actor:self:user-123",
-    scope_profile: "actor_client_v1",
-    policy_epoch: CURRENT_POLICY_EPOCH,
-    ttlSeconds: 3600,
-  });
-}
-
 function startFrame(conversationId = "conversation-123"): string {
   return JSON.stringify({
     type: "start",
@@ -161,33 +142,6 @@ async function waitForOpen(ws: WebSocket, timeoutMs = 2000): Promise<void> {
   });
 }
 
-async function waitForClose(ws: WebSocket, timeoutMs = 2000): Promise<void> {
-  if (ws.readyState === WebSocket.CLOSED) {
-    return;
-  }
-  await new Promise<void>((resolve, reject) => {
-    const timer = setTimeout(() => {
-      cleanup();
-      reject(new Error("Timed out waiting for WebSocket close"));
-    }, timeoutMs);
-    const cleanup = () => {
-      clearTimeout(timer);
-      ws.removeEventListener("close", onClose);
-      ws.removeEventListener("error", onError);
-    };
-    const onClose = () => {
-      cleanup();
-      resolve();
-    };
-    const onError = () => {
-      cleanup();
-      reject(new Error("WebSocket close failed"));
-    };
-    ws.addEventListener("close", onClose);
-    ws.addEventListener("error", onError);
-  });
-}
-
 async function waitForJsonFrame(
   ws: WebSocket,
   timeoutMs = 2000,
@@ -241,6 +195,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
   let baseUrl: string;
   let wsBaseUrl: string;
   let clients: WebSocket[];
+  let restoreAuthEnv: () => void;
 
   beforeAll(() => {
     shellMocksActive = true;
@@ -251,7 +206,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
   });
 
   beforeEach(async () => {
-    delete process.env.DISABLE_HTTP_AUTH;
+    restoreAuthEnv = requireHttpAuth();
     resolveStreamingTranscriberImpl = async () => createResolvedTranscriber();
     resolveStreamingTranscriberMock.mockClear();
     resolvedTranscribers.length = 0;
@@ -268,11 +223,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
       closeClient(client);
     }
     await server.stop();
-    if (savedAuthEnv.DISABLE_HTTP_AUTH === undefined) {
-      delete process.env.DISABLE_HTTP_AUTH;
-    } else {
-      process.env.DISABLE_HTTP_AUTH = savedAuthEnv.DISABLE_HTTP_AUTH;
-    }
+    restoreAuthEnv();
   });
 
   function openLiveVoiceClient(token = mintGatewayToken()): WebSocket {
@@ -284,21 +235,14 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
   }
 
   test("rejects unauthorized upgrades before creating a WebSocket", async () => {
-    const baseHeaders = {
-      Upgrade: "websocket",
-      Connection: "Upgrade",
-      "Sec-WebSocket-Key": "dGhlIHNhbXBsZSBub25jZQ==",
-      "Sec-WebSocket-Version": "13",
-    };
-
     const missingToken = await fetch(`${baseUrl}/v1/live-voice`, {
-      headers: baseHeaders,
+      headers: upgradeHeaders,
     });
     expect(missingToken.status).toBe(401);
 
     const actorToken = await fetch(
       `${baseUrl}/v1/live-voice?token=${mintActorToken()}`,
-      { headers: baseHeaders },
+      { headers: upgradeHeaders },
     );
     expect(actorToken.status).toBe(401);
 
@@ -306,7 +250,7 @@ describe("RuntimeHttpServer live voice WebSocket shell", () => {
       `${baseUrl}/v1/live-voice?token=${mintGatewayToken()}`,
       {
         headers: {
-          ...baseHeaders,
+          ...upgradeHeaders,
           Origin: "https://external.example.com",
         },
       },

package/src/live-voice/__tests__/session-controls.test.ts

@@ -0,0 +1,156 @@
+import { describe, expect, test } from "bun:test";
+
+import type { VoiceProgressConfig } from "../../config/schemas/voice.js";
+import {
+  FEWER_UPDATES_INTERVAL_MS,
+  progressConfigForCadence,
+  requestedSessionControl,
+  sessionControlTeaching,
+} from "../session-controls.js";
+
+const CONFIG: VoiceProgressConfig = {
+  enabled: true,
+  opsThreshold: 3,
+  idleIntervalMs: 5_000,
+  maxSilenceMs: 35_000,
+  longOpMs: 15_000,
+  minGapMs: 6_000,
+  generationTimeoutMs: 1_500,
+};
+
+describe("progressConfigForCadence", () => {
+  test("normal is the configured cadence, untouched", () => {
+    expect(progressConfigForCadence(CONFIG, "normal")).toBe(CONFIG);
+  });
+
+  test("fewer silences tool activity and stretches the silence tick", () => {
+    const fewer = progressConfigForCadence(CONFIG, "fewer");
+
+    expect(fewer.opsThreshold).toBe(Number.MAX_SAFE_INTEGER);
+    expect(fewer.longOpMs).toBe(Number.MAX_SAFE_INTEGER);
+    expect(fewer.idleIntervalMs).toBe(FEWER_UPDATES_INTERVAL_MS);
+    expect(fewer.maxSilenceMs).toBe(FEWER_UPDATES_INTERVAL_MS);
+    // Narration stays on: a very long silence still proves the call is alive.
+    expect(fewer.enabled).toBe(true);
+  });
+
+  test("fewer never speaks more often than the configured heartbeat", () => {
+    const fewer = progressConfigForCadence(
+      { ...CONFIG, idleIntervalMs: 90_000, maxSilenceMs: 120_000 },
+      "fewer",
+    );
+
+    expect(fewer.idleIntervalMs).toBe(120_000);
+    expect(fewer.maxSilenceMs).toBe(120_000);
+  });
+});
+
+describe("requestedSessionControl", () => {
+  test("the update cadence needs no client declaration", () => {
+    expect(requestedSessionControl("Sure. [UPDATES:FEWER]", [])).toEqual({
+      action: "updates",
+      cadence: "fewer",
+    });
+  });
+
+  test("task stop needs no client declaration", () => {
+    expect(requestedSessionControl("Okay. [TASK:STOP]", [])).toEqual({
+      action: "task_stop",
+    });
+  });
+
+  test("a client control needs the client's declaration", () => {
+    expect(requestedSessionControl("Bye. [END_CALL]", [])).toBeNull();
+    expect(requestedSessionControl("Bye. [END_CALL]", ["end"])).toEqual({
+      action: "end",
+    });
+  });
+});
+
+describe("sessionControlTeaching", () => {
+  test("teaches the update cadence and only the declared client controls", () => {
+    const teaching = sessionControlTeaching(["mute"], {});
+
+    expect(teaching).toContain("[UPDATES:FEWER]");
+    expect(teaching).toContain("[MUTE]");
+    expect(teaching).not.toContain("[END_CALL]");
+    expect(teaching).not.toContain("[TASK:STOP]");
+    expect(teaching).toContain("Never emit any other bracketed marker.");
+  });
+
+  test("teaches task stop only while an unfinished task is pending", () => {
+    const teaching = sessionControlTeaching(
+      [],
+      {},
+      { unfinishedTaskPending: true },
+    );
+
+    expect(teaching).toContain("[TASK:STOP]");
+    expect(teaching).toContain("not when they interrupt with a question");
+  });
+
+  test("a device that can show both asks which one a bare look means", () => {
+    const teaching = sessionControlTeaching(["look_screen", "look_camera"], {});
+
+    expect(teaching).toContain("[LOOK:SCREEN]");
+    expect(teaching).toContain("[LOOK:CAMERA]");
+    expect(teaching).toContain("ask which one instead of guessing");
+  });
+
+  test("a device that can look can be told to stop", () => {
+    const teaching = sessionControlTeaching(["look_screen", "look_stop"], {});
+
+    expect(teaching).toContain("[LOOK:STOP]");
+    expect(
+      requestedSessionControl("Stopping. [LOOK:STOP]", ["look_stop"]),
+    ).toEqual({ action: "look_stop" });
+  });
+
+  test("a device that can show one look needs no question", () => {
+    const teaching = sessionControlTeaching(["look_camera"], {});
+
+    expect(teaching).toContain("[LOOK:CAMERA]");
+    expect(teaching).not.toContain("[LOOK:SCREEN]");
+    expect(teaching).not.toContain("ask which one");
+    expect(teaching).not.toContain("cannot turn on a screen share");
+  });
+
+  // The silent failure in the report this exists for: asked to look, the
+  // assistant neither looked nor said it could not.
+  test("a device that can show neither says so rather than pretending", () => {
+    expect(sessionControlTeaching(["end"], {})).toContain(
+      "This call cannot turn on a screen share or the camera.",
+    );
+  });
+
+  // A client that sends a fresh frame for every look lets the session answer
+  // it, so the reply asking for the look only acknowledges, and asking again is
+  // how a share already running gets looked at as it is now.
+  test("a client that sends look frames is taught the look is answered for it", () => {
+    const teaching = sessionControlTeaching(
+      ["look_screen", "look_camera"],
+      {},
+      { lookFrames: true },
+    );
+
+    expect(teaching).toContain("[LOOK:SCREEN]");
+    expect(teaching).toContain(
+      "Use it even when their screen is already shared with you",
+    );
+    expect(teaching).toContain("Use it even when the camera is already on");
+    expect(teaching).not.toContain("take it from their next words");
+  });
+
+  test("a client that sends no look frames keeps the next-words look", () => {
+    const teaching = sessionControlTeaching(["look_screen"], {});
+
+    expect(teaching).toContain("take it from their next words");
+    expect(teaching).not.toContain("Use it even when");
+  });
+
+  test("the front-door leg keeps its verdict tokens", () => {
+    expect(sessionControlTeaching(["end"], { frontDoor: true })).not.toContain(
+      "Never emit any other bracketed marker.",
+    );
+  });
+});

package/src/live-voice/live-voice-metrics.ts

@@ -511,6 +511,15 @@ export class LiveVoiceMetricsCollector {
         turnId: turn.turnId,
         finishReason,
         ...aggregateFieldsForTurn(snapshotTurn(turn)),
+        // Stated even at zero, unlike the telemetry field this overrides.
+        // Zero is the single most informative value this count takes: it is
+        // what a turn asked for fewer updates looks like, and equally what a
+        // narrator that is failing every attempt looks like. Left absent, the
+        // two read identically to anyone reading the log, and "the feature is
+        // off" becomes indistinguishable from "the feature did its job".
+        // The wire payload keeps the omission (see optionalTurnFields), so
+        // turns that never engage narration are unchanged for telemetry.
+        progressUpdatesSpoken: turn.progressUpdatesSpoken,
       },
       "Live voice turn latency",
     );

package/src/live-voice/live-voice-photo.ts

@@ -237,6 +237,7 @@ async function enqueueStandaloneImagePersist(
         [ticket.attachmentId],
         ticket.content,
         uuidv7(),
+        kind,
       );
       return { ok: false };
     }
@@ -389,11 +390,18 @@ function reclaimOrDefer(
   attachmentIds: readonly string[],
   content: string,
   messageId: string,
+  kind: "photo" | "sight_frame",
 ): void {
   if (reclaimDroppedFrame(attachmentIds)) {
     return;
   }
-  deferFrameReclaimDecision(conversationId, messageId, attachmentIds, content);
+  deferFrameReclaimDecision(
+    conversationId,
+    messageId,
+    attachmentIds,
+    content,
+    kind,
+  );
 }
 
 /** Conversations with a standalone-image persist still in flight. */
@@ -429,6 +437,8 @@ interface PendingFrameReclaim {
   attachmentIds: readonly string[];
   /** Row text, so a frame found to have landed can still be announced. */
   content: string;
+  /** The echo shape used if a later recheck finds the row. */
+  kind: "photo" | "sight_frame";
   attempts: number;
 }
 
@@ -465,12 +475,14 @@ function deferFrameReclaimDecision(
   messageId: string,
   attachmentIds: readonly string[],
   content: string,
+  kind: "photo" | "sight_frame",
 ): void {
   pendingFrameReclaims.push({
     conversationId,
     messageId,
     attachmentIds,
     content,
+    kind,
     attempts: 0,
   });
   scheduleFrameReclaimRecheck();
@@ -521,6 +533,7 @@ function announceDeferredImage(pending: PendingFrameReclaim): void {
       pending.conversationId,
       pending.content,
       pending.messageId,
+      pending.kind,
     );
   } catch (err) {
     log.warn(
@@ -700,6 +713,7 @@ function persistStandaloneImage(
         uuidv7(),
         [attachmentId],
         persistOptions.content,
+        kind,
       );
     }
     return Promise.resolve({ ok: false });
@@ -715,6 +729,7 @@ function persistStandaloneImage(
         [attachmentId],
         persistOptions.content,
         uuidv7(),
+        kind,
       );
     }
     return Promise.resolve({ ok: false });
@@ -754,7 +769,7 @@ function dropReplacedImage(
     "Standalone image dropped: its conversation was replaced before the write",
   );
   if (kind === "sight_frame") {
-    reclaimOrDefer(conversationId, [attachmentId], content, uuidv7());
+    reclaimOrDefer(conversationId, [attachmentId], content, uuidv7(), kind);
   }
   return { ok: false };
 }
@@ -806,7 +821,7 @@ async function writeStandaloneImage(
         "Standalone image dropped: its conversation was deleted while it waited",
       );
       if (kind === "sight_frame") {
-        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7());
+        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7(), kind);
       }
       return { ok: false };
     }
@@ -831,7 +846,7 @@ async function writeStandaloneImage(
         "Standalone image timed out waiting for the conversation to go idle",
       );
       if (kind === "sight_frame") {
-        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7());
+        reclaimOrDefer(conversationId, [attachmentId], content, uuidv7(), kind);
       }
       return { ok: false };
     }
@@ -882,7 +897,7 @@ async function writeStandaloneImage(
       // of.
       conversation.markHistoryStaleForForeignScope(persistOptions.trustContext);
 
-      announcePersistedImage(conversationId, content, persisted.id);
+      announcePersistedImage(conversationId, content, persisted.id, kind);
 
       return {
         ok: true,
@@ -930,7 +945,7 @@ async function writeStandaloneImage(
       // `ensureActorScopedHistory` reloads instead of reusing what it holds.
       findConversation(conversationId)?.markHistoryStale();
       try {
-        announcePersistedImage(conversationId, content, requestId);
+        announcePersistedImage(conversationId, content, requestId, kind);
       } catch (announceErr) {
         log.warn(
           { err: announceErr, conversationId, messageId: requestId },
@@ -946,6 +961,7 @@ async function writeStandaloneImage(
           [attachmentId, ...strandedClones],
           content,
           requestId,
+          kind,
         );
       } else {
         // The store would not say whether the row landed. The refusal below
@@ -957,6 +973,7 @@ async function writeStandaloneImage(
           requestId,
           [attachmentId, ...strandedClones],
           content,
+          kind,
         );
       }
     }
@@ -974,12 +991,14 @@ function announcePersistedImage(
   conversationId: string,
   text: string,
   messageId: string,
+  kind: "photo" | "sight_frame",
 ): void {
   broadcastMessage({
     type: "user_message_echo",
     text,
     conversationId,
     messageId,
+    ...(kind === "sight_frame" ? { cameraFrame: true as const } : {}),
   });
   recordConversationPersistedSeq(conversationId, getCurrentSeq());
   publishConversationMessagesChanged(conversationId);