@vellumai/assistant 0.11.0 → 0.11.1-staging.2

package/src/live-voice/__tests__/protocol.test.ts

@@ -24,7 +24,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     );
 
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
 
     expect(result.frame).toEqual({
       type: "start",
@@ -43,7 +45,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     );
 
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
 
     expect(result.frame).toEqual({
       type: "audio",
@@ -77,7 +81,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
       JSON.stringify({ type: "update_config", silenceThresholdMs: 900 }),
     );
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
     expect(result.frame).toEqual({
       type: "update_config",
       silenceThresholdMs: 900,
@@ -102,7 +108,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
       [field]: badValue,
     });
     expect(result.ok).toBe(false);
-    if (result.ok) return;
+    if (result.ok) {
+      return;
+    }
     expect(result.error).toMatchObject({
       code: "invalid_field",
       field,
@@ -114,7 +122,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     const result = parseLiveVoiceClientTextFrame("{");
 
     expect(result.ok).toBe(false);
-    if (result.ok) return;
+    if (result.ok) {
+      return;
+    }
 
     expect(result.error.code).toBe("invalid_json");
   });
@@ -123,7 +133,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     const result = parseLiveVoiceClientTextFrame("[]");
 
     expect(result.ok).toBe(false);
-    if (result.ok) return;
+    if (result.ok) {
+      return;
+    }
 
     expect(result.error).toMatchObject({
       code: "invalid_frame",
@@ -136,7 +148,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     );
 
     expect(result.ok).toBe(false);
-    if (result.ok) return;
+    if (result.ok) {
+      return;
+    }
 
     expect(result.error).toMatchObject({
       code: "unknown_type",
@@ -202,7 +216,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     });
 
     expect(result.ok).toBe(false);
-    if (result.ok) return;
+    if (result.ok) {
+      return;
+    }
 
     expect(result.error).toMatchObject({
       code: "invalid_field",
@@ -324,7 +340,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     );
 
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
     expect(result.frame).toMatchObject({
       type: "start",
       silenceThresholdMs: 1500,
@@ -342,7 +360,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     );
 
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
     expect(result.frame).toMatchObject({ bargeInMinSpeechMs: 0 });
   });
 
@@ -355,7 +375,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     );
 
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
     expect("silenceThresholdMs" in result.frame).toBe(false);
     expect("bargeInMinSpeechMs" in result.frame).toBe(false);
   });
@@ -377,7 +399,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
       });
 
       expect(result.ok).toBe(false);
-      if (result.ok) return;
+      if (result.ok) {
+        return;
+      }
       expect(result.error).toMatchObject({
         code: "invalid_field",
         field,
@@ -396,7 +420,9 @@ describe("parseLiveVoiceClientTextFrame", () => {
     });
 
     expect(result.ok).toBe(false);
-    if (result.ok) return;
+    if (result.ok) {
+      return;
+    }
 
     expect(result.error).toMatchObject({
       code: "missing_required_field",
@@ -412,7 +438,9 @@ describe("parseLiveVoiceBinaryAudioFrame", () => {
     const result = parseLiveVoiceBinaryAudioFrame(bytes.buffer);
 
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
 
     expect(result.frame.type).toBe("binary_audio");
     expect(Array.from(result.frame.data)).toEqual([1, 2, 3]);
@@ -423,7 +451,9 @@ describe("parseLiveVoiceBinaryAudioFrame", () => {
     const result = parseLiveVoiceBinaryAudioFrame(source.subarray(1, 3));
 
     expect(result.ok).toBe(true);
-    if (!result.ok) return;
+    if (!result.ok) {
+      return;
+    }
 
     expect(Array.from(result.frame.data)).toEqual([8, 7]);
   });

package/src/live-voice/__tests__/runtime-websocket-shell.test.ts

@@ -135,7 +135,9 @@ function startFrame(conversationId = "conversation-123"): string {
 }
 
 async function waitForOpen(ws: WebSocket, timeoutMs = 2000): Promise<void> {
-  if (ws.readyState === WebSocket.OPEN) return;
+  if (ws.readyState === WebSocket.OPEN) {
+    return;
+  }
   await new Promise<void>((resolve, reject) => {
     const timer = setTimeout(() => {
       cleanup();
@@ -160,7 +162,9 @@ async function waitForOpen(ws: WebSocket, timeoutMs = 2000): Promise<void> {
 }
 
 async function waitForClose(ws: WebSocket, timeoutMs = 2000): Promise<void> {
-  if (ws.readyState === WebSocket.CLOSED) return;
+  if (ws.readyState === WebSocket.CLOSED) {
+    return;
+  }
   await new Promise<void>((resolve, reject) => {
     const timer = setTimeout(() => {
       cleanup();

package/src/live-voice/live-activity-reporter.ts

@@ -0,0 +1,180 @@
+/**
+ * Reports a live-voice session's phase to the platform, which pushes it to the
+ * iOS Live Activity over APNs.
+ *
+ * ## Why the server reports a phase the client already knows
+ *
+ * Every phase the island shows is derived, client-side, from frames this
+ * session sends: `utterance_ended` → transcribing, `stt_final`/`thinking` →
+ * thinking, the first `tts_audio` → speaking. The web layer then pushes that
+ * phase to ActivityKit itself, which works perfectly — right up to the moment
+ * the app is backgrounded, which is the only moment the Lock Screen and the
+ * Dynamic Island are on screen at all. iOS throttles and eventually suspends
+ * that web view, and a suspended web view cannot report anything.
+ *
+ * So the same derivation runs here, on the side of the connection that stays
+ * up. This is deliberately a *mirror* of the client's mapping rather than a new
+ * source of truth: {@link phaseForFrame} and the client's frame handlers must
+ * agree, and the client's remain authoritative for anything they can observe
+ * that this cannot (a reconnect, a mute, whether TTS audio is actually
+ * audible).
+ *
+ * ## Everything here is best-effort
+ *
+ * A voice session must be entirely unaffected by this. Every failure is logged
+ * and dropped: no retry, no backpressure, and never a throw into the session's
+ * frame path. A missed island update costs a stale label for a few seconds; a
+ * throw here would cost the conversation.
+ */
+
+import { VellumPlatformClient } from "../platform/client.js";
+import { getLogger } from "../util/logger.js";
+import type { LiveVoiceServerFramePayload } from "./protocol.js";
+
+const log = getLogger("live-activity-reporter");
+
+/** The phases an iOS Live Activity can render. Mirrors the client's union. */
+export type LiveActivityPhase =
+  | "connecting"
+  | "listening"
+  | "transcribing"
+  | "thinking"
+  | "speaking"
+  | "ending";
+
+/**
+ * The phase a frame puts the session into, or `null` for frames that do not
+ * move it.
+ *
+ * `frame` is typed as the protocol's own payload union rather than a bare
+ * string, so a `case` for a frame that does not exist is a compile error. The
+ * first version of this function matched `"utterance_ended"` — a frame no part
+ * of the system emits — and silently never fired.
+ *
+ * **This mirrors the client's handlers** in
+ * `clients/web/src/domains/chat/voice/live-voice/use-live-voice.ts`, which stay
+ * authoritative. Frames absent here are not oversights: most say nothing about
+ * the phase (`stt_partial` streams inside `listening`, and every `tts_audio`
+ * after the first is already `speaking`).
+ *
+ * `lastPhase` is a parameter because two of the client's rules are stateful,
+ * and dropping them would put wrong wording on a Lock Screen the user cannot
+ * correct.
+ */
+export function phaseForFrame(
+  frame: LiveVoiceServerFramePayload,
+  lastPhase: LiveActivityPhase | null,
+): LiveActivityPhase | null {
+  switch (frame.type) {
+    case "utterance_end":
+      return "transcribing";
+    case "utterance_discarded":
+      // The utterance held no usable speech, so the turn never happens and the
+      // session drops back to listening.
+      return "listening";
+    case "stt_final":
+      // **Not every final means the assistant is thinking.** Semantic
+      // endpointing can hold an utterance open past a final — the session
+      // suppresses `utterance_end`, and the floor is still the user's — so
+      // only a *closed* utterance advances. An empty final never starts a turn
+      // either; its utterance is about to be discarded. Both conditions are
+      // the client's, restated here only because this side cannot observe the
+      // store: see the `sttFinal` handler in `use-live-voice.ts`.
+      if (lastPhase !== "transcribing" || frame.text.trim().length === 0) {
+        return null;
+      }
+      return "thinking";
+    case "thinking":
+      // Unconditional, unlike `stt_final`: the session sends this frame
+      // precisely when it has committed to a turn.
+      return "thinking";
+    case "tts_audio":
+      return "speaking";
+    case "tts_done":
+      // The assistant stopped talking; the floor is the user's again.
+      return "listening";
+    default:
+      return null;
+  }
+}
+
+/**
+ * Pushes phase changes for one session, de-duplicating repeats.
+ *
+ * The de-duplication is not an optimization. `tts_audio` arrives per audio
+ * chunk — many times a second — and ActivityKit rate-limits updates and drops
+ * the overflow silently, so a reporter that forwarded every frame would spend
+ * the session's whole update budget restating `speaking` and have none left
+ * for the phase the user is waiting to see.
+ */
+export class LiveActivityReporter {
+  private lastPhase: LiveActivityPhase | null = null;
+  private ended = false;
+
+  constructor(private readonly conversationId: string) {}
+
+  /**
+   * Report the phase a frame implies, if it changed. Fire-and-forget: callers
+   * are on the session's send path and must not await this.
+   */
+  report(frame: LiveVoiceServerFramePayload): void {
+    const phase = phaseForFrame(frame, this.lastPhase);
+    if (phase === null || phase === this.lastPhase || this.ended) {
+      return;
+    }
+    this.lastPhase = phase;
+    void this.dispatch(phase, "update");
+  }
+
+  /**
+   * Retire the activity as the session ends.
+   *
+   * Latched, because a session can end more than one way — a clean hangup, a
+   * dropped socket, a process shutdown — and the platform deletes the
+   * registration on the first one. Sending a second would push at an activity
+   * that is already gone.
+   */
+  end(): void {
+    if (this.ended) {
+      return;
+    }
+    this.ended = true;
+    void this.dispatch("ending", "end");
+  }
+
+  /** `protected` so a test can observe what would be sent without sending it. */
+  protected async dispatch(
+    phase: LiveActivityPhase,
+    event: "update" | "end",
+  ): Promise<void> {
+    try {
+      const client = await VellumPlatformClient.create();
+      if (!client?.platformAssistantId) {
+        return;
+      }
+      const path = `/v1/assistants/${encodeURIComponent(
+        client.platformAssistantId,
+      )}/live-activity/dispatch/`;
+      const response = await client.fetch(path, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          conversation_id: this.conversationId,
+          phase,
+          event,
+        }),
+      });
+      if (!response.ok) {
+        log.debug(
+          { status: response.status, phase, event },
+          "Live Activity dispatch rejected",
+        );
+      }
+    } catch (err) {
+      // Deliberately not retried. By the time a retry landed the phase would
+      // have moved on, and re-sending a stale one is worse than sending
+      // nothing — `aps.timestamp` would let it through as newer.
+      log.debug({ err, phase, event }, "Live Activity dispatch failed");
+    }
+  }
+}

package/src/live-voice/live-voice-archive.ts

@@ -191,7 +191,9 @@ function buildFilenameStem(input: {
 
 function extensionForAudioMimeType(mimeType: string): string {
   const mapped = AUDIO_EXTENSION_BY_MIME_TYPE[mimeType];
-  if (mapped) return mapped;
+  if (mapped) {
+    return mapped;
+  }
   const subtype = mimeType.slice("audio/".length);
   return sanitizeFilenamePart(subtype.replace(/^x-/, "")) || "audio";
 }
@@ -218,7 +220,9 @@ function isLiveVoiceRole(value: unknown): value is LiveVoiceAudioArchiveRole {
 function isArtifactMetadata(
   value: unknown,
 ): value is LiveVoiceAudioArtifactMetadata {
-  if (!isRecord(value)) return false;
+  if (!isRecord(value)) {
+    return false;
+  }
   return (
     value.source === LIVE_VOICE_AUDIO_SOURCE &&
     typeof value.archiveKey === "string" &&
@@ -241,7 +245,9 @@ function readMessageMetadata(
     `SELECT metadata FROM messages WHERE id = ?`,
     messageId,
   );
-  if (!row) return "not_found";
+  if (!row) {
+    return "not_found";
+  }
 
   if (!row.metadata) {
     return { metadata: {}, artifacts: [] };
@@ -337,7 +343,9 @@ function persistArtifactMetadata(
   artifact: LiveVoiceAudioArtifactMetadata,
 ): boolean {
   const state = readMessageMetadata(messageId);
-  if (state === "not_found" || state === "invalid_metadata") return false;
+  if (state === "not_found" || state === "invalid_metadata") {
+    return false;
+  }
 
   const nextArtifacts = [
     ...state.artifacts.filter(
@@ -361,7 +369,9 @@ function persistArtifactMetadata(
 function sanitizeOptionalPositiveNumber(
   value: number | undefined,
 ): number | undefined {
-  if (value == null) return undefined;
+  if (value == null) {
+    return undefined;
+  }
   return Number.isFinite(value) && value > 0 ? value : undefined;
 }
 
@@ -462,7 +472,9 @@ export function archiveLiveVoiceAudioArtifact(
   input: ArchiveLiveVoiceAudioInput,
 ): LiveVoiceAudioArchiveResult {
   const validated = validateInput(input);
-  if ("type" in validated) return validated;
+  if ("type" in validated) {
+    return validated;
+  }
 
   const state = readMessageMetadata(input.messageId);
   if (state === "not_found") {

package/src/live-voice/live-voice-metrics.ts

@@ -277,8 +277,12 @@ export class LiveVoiceMetricsCollector {
     let earliestMs = startedAtMs;
     for (const field of SEEDABLE_MARK_FIELDS) {
       const value = seeds[field];
-      if (value === undefined || !Number.isFinite(value)) continue;
-      if (turn.timestamps[field] !== null) continue;
+      if (value === undefined || !Number.isFinite(value)) {
+        continue;
+      }
+      if (turn.timestamps[field] !== null) {
+        continue;
+      }
       const seededMs = Math.min(value, startedAtMs);
       turn.timestamps[field] = seededMs;
       earliestMs = Math.min(earliestMs, seededMs);
@@ -607,8 +611,12 @@ function frontModelFields(
 }
 
 function normalizeRecentTurnLimit(limit: number | undefined): number {
-  if (limit === undefined) return DEFAULT_RECENT_TURN_LIMIT;
-  if (!Number.isFinite(limit) || limit < 1) return DEFAULT_RECENT_TURN_LIMIT;
+  if (limit === undefined) {
+    return DEFAULT_RECENT_TURN_LIMIT;
+  }
+  if (!Number.isFinite(limit) || limit < 1) {
+    return DEFAULT_RECENT_TURN_LIMIT;
+  }
   return Math.floor(limit);
 }
 
@@ -617,11 +625,15 @@ function selectTurnForAggregate(
   turnId: string | undefined,
 ): LiveVoiceTurnMetrics | null {
   if (turnId !== undefined) {
-    if (snapshot.activeTurn?.turnId === turnId) return snapshot.activeTurn;
+    if (snapshot.activeTurn?.turnId === turnId) {
+      return snapshot.activeTurn;
+    }
     const matchingRecentTurn = snapshot.recentTurns.find(
       (turn) => turn.turnId === turnId,
     );
-    if (matchingRecentTurn) return matchingRecentTurn;
+    if (matchingRecentTurn) {
+      return matchingRecentTurn;
+    }
   }
 
   return (
@@ -747,12 +759,16 @@ function percentile(
   sortedValues: number[],
   percentileValue: number,
 ): number | null {
-  if (sortedValues.length === 0) return null;
+  if (sortedValues.length === 0) {
+    return null;
+  }
   const index = Math.ceil(sortedValues.length * percentileValue) - 1;
   return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)];
 }
 
 function duration(startMs: number | null, endMs: number | null): number | null {
-  if (startMs === null || endMs === null) return null;
+  if (startMs === null || endMs === null) {
+    return null;
+  }
   return Math.max(0, endMs - startMs);
 }