@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

package/src/live-voice/live-voice-metrics.ts

@@ -15,10 +15,10 @@ export type LiveVoiceMetricsEvent =
   | "ptt_release"
   | "utterance_end"
   | "endpoint_decision"
+  | "endpoint_commit"
   | "barge_in"
   | "final_transcript"
   | "first_assistant_delta"
-  | "ack_spoken"
   | "progress_spoken"
   | "first_tts_audio"
   | "turn_completed"
@@ -29,15 +29,19 @@ export type LiveVoiceMetricsEvent =
 // front-door leg either holds the utterance open or releases it to the turn.
 export type VoiceEndpointAction = "release" | "hold";
 
+// Which decider produced the endpoint outcome: the speculative front-door
+// leg, or the STT provider's own turn-detection stream.
+export type VoiceEndpointSource = "front-door" | "provider";
+
+const DEFAULT_ENDPOINT_SOURCE: VoiceEndpointSource = "front-door";
+
 // Semantic-endpointing decision on a silence boundary.
 interface LiveVoiceEndpointDecisionMark {
   action: VoiceEndpointAction;
   latencyMs: number;
+  source?: VoiceEndpointSource;
 }
 
-// Which floor-holding ack actually spoke during a turn.
-export type LiveVoiceSpokenAckKind = "first_delta" | "tool_use";
-
 type LiveVoiceTurnStatus = "active" | "completed" | "cancelled";
 
 interface LiveVoiceMetricsCollectorOptions {
@@ -117,12 +121,16 @@ interface LiveVoiceTurnMetrics {
   cancellationReason: string | null;
   timestamps: LiveVoiceTurnTimestamps;
   durations: LiveVoiceTurnDurations;
-  // Present only when the semantic-endpointing decider was consulted for the
-  // turn / when an ack actually spoke / when a progress narration spoke, so
+  // End of turn measured from the local VAD speech-stop mark to the commit,
+  // recorded on every committed turn whichever decider owned the boundary.
+  // Absent on a turn that never committed, and in push-to-talk mode, where
+  // there is no local speech-stop mark to anchor to.
+  endpointCommitLatencyMs?: number;
+  // Present only when endpointing or progress narration engaged, so
   // turns that never touch the features carry no trace of them.
   endpointHoldCount?: number;
   endpointDecisionMaxLatencyMs?: number;
-  ackSpoken?: LiveVoiceSpokenAckKind;
+  endpointDecisionSource?: VoiceEndpointSource;
   progressUpdatesSpoken?: number;
 }
 
@@ -165,11 +173,13 @@ interface LiveVoiceMetricsAggregateFields {
   ttsFirstAudioMs: number | null;
   roundTripMs: number | null;
   totalMs: number | null;
-  // Optional so metrics frames stay byte-identical when the front-model
-  // features never engaged (see the matching fields on LiveVoiceTurnMetrics).
+  // Optional so metrics frames stay byte-identical when the turn never
+  // committed and when the front-model features never engaged (see the
+  // matching fields on LiveVoiceTurnMetrics).
+  endpointCommitLatencyMs?: number;
   endpointHoldCount?: number;
   endpointDecisionMaxLatencyMs?: number;
-  ackSpoken?: LiveVoiceSpokenAckKind;
+  endpointDecisionSource?: VoiceEndpointSource;
   progressUpdatesSpoken?: number;
 }
 
@@ -187,11 +197,14 @@ interface MutableTurn {
   status: LiveVoiceTurnStatus;
   cancellationReason: string | null;
   timestamps: LiveVoiceTurnTimestamps;
+  // Null means the turn never committed (see LiveVoiceTurnMetrics).
+  endpointCommitLatencyMs: number | null;
   endpointHoldCount: number;
   // Doubles as the "decider was consulted" latch: null means no endpoint
   // decision was ever recorded for the turn.
   endpointDecisionMaxLatencyMs: number | null;
-  ackSpoken: LiveVoiceSpokenAckKind | null;
+  // The decider behind the turn's most recent endpoint decision.
+  endpointDecisionSource: VoiceEndpointSource | null;
   progressUpdatesSpoken: number;
 }
 
@@ -255,9 +268,10 @@ export class LiveVoiceMetricsCollector {
         completedAtMs: null,
         cancelledAtMs: null,
       },
+      endpointCommitLatencyMs: null,
       endpointHoldCount: 0,
       endpointDecisionMaxLatencyMs: null,
-      ackSpoken: null,
+      endpointDecisionSource: null,
       progressUpdatesSpoken: 0,
     };
     this.applySeedMarks(this.activeTurn, seedMarks);
@@ -340,25 +354,24 @@ export class LiveVoiceMetricsCollector {
     if (decision.action === "hold") {
       turn.endpointHoldCount += 1;
     }
-    const latencyMs = Number.isFinite(decision.latencyMs)
-      ? Math.max(0, decision.latencyMs)
-      : 0;
     turn.endpointDecisionMaxLatencyMs = Math.max(
       turn.endpointDecisionMaxLatencyMs ?? 0,
-      latencyMs,
+      normalizeLatencyMs(decision.latencyMs),
     );
+    turn.endpointDecisionSource = decision.source ?? DEFAULT_ENDPOINT_SOURCE;
     return this.emit("endpoint_decision", turn.turnId);
   }
 
-  markAckSpoken(
+  // First-wins, since an utterance commits once.
+  markEndpointCommit(
     turnId: string | undefined,
-    kind: LiveVoiceSpokenAckKind,
+    latencyMs: number,
   ): LiveVoiceMetricsFrame {
     const turn = this.ensureActiveTurn(turnId);
-    if (turn.ackSpoken === null) {
-      turn.ackSpoken = kind;
+    if (turn.endpointCommitLatencyMs === null) {
+      turn.endpointCommitLatencyMs = normalizeLatencyMs(latencyMs);
     }
-    return this.emit("ack_spoken", turn.turnId);
+    return this.emit("endpoint_commit", turn.turnId);
   }
 
   // A counter, not a first-wins mark: every spoken progress narration bumps
@@ -574,38 +587,45 @@ function aggregateFieldsForTurn(
     ttsFirstAudioMs: turn.durations.firstAssistantDeltaToFirstTtsAudioMs,
     roundTripMs: turn.durations.roundTripMs,
     totalMs: turn.durations.totalTurnDurationMs,
-    ...frontModelFields(turn),
+    ...optionalTurnFields(turn),
   };
 }
 
-// Shared optional front-model fields for turn snapshots and aggregate
-// frame fields: absent unless the endpoint decider was consulted / an ack
-// spoke / a progress narration spoke, so turns that never touch the
-// features are unchanged.
-function frontModelFields(
+// Shared optional fields for turn snapshots and aggregate frame fields: the
+// commit latency is absent unless the turn committed against a local
+// speech-stop mark, and the front-model fields are absent unless the endpoint
+// decider was consulted or a progress narration spoke, so turns
+// that never touch the features are unchanged.
+function optionalTurnFields(
   // Accepts both MutableTurn (null = unset) and snapshot (absent = unset).
   turn: {
+    endpointCommitLatencyMs?: number | null;
     endpointHoldCount?: number | null;
     endpointDecisionMaxLatencyMs?: number | null;
-    ackSpoken?: LiveVoiceSpokenAckKind | null;
+    endpointDecisionSource?: VoiceEndpointSource | null;
     progressUpdatesSpoken?: number | null;
   },
 ): Pick<
   LiveVoiceTurnMetrics,
+  | "endpointCommitLatencyMs"
   | "endpointHoldCount"
   | "endpointDecisionMaxLatencyMs"
-  | "ackSpoken"
+  | "endpointDecisionSource"
   | "progressUpdatesSpoken"
 > {
   const progressUpdatesSpoken = turn.progressUpdatesSpoken ?? 0;
   return {
+    ...(turn.endpointCommitLatencyMs != null
+      ? { endpointCommitLatencyMs: turn.endpointCommitLatencyMs }
+      : {}),
     ...(turn.endpointDecisionMaxLatencyMs != null
       ? {
           endpointHoldCount: turn.endpointHoldCount ?? 0,
           endpointDecisionMaxLatencyMs: turn.endpointDecisionMaxLatencyMs,
+          endpointDecisionSource:
+            turn.endpointDecisionSource ?? DEFAULT_ENDPOINT_SOURCE,
         }
       : {}),
-    ...(turn.ackSpoken != null ? { ackSpoken: turn.ackSpoken } : {}),
     ...(progressUpdatesSpoken > 0 ? { progressUpdatesSpoken } : {}),
   };
 }
@@ -649,9 +669,10 @@ function cloneMutableTurn(turn: MutableTurn): MutableTurn {
     status: turn.status,
     cancellationReason: turn.cancellationReason,
     timestamps: { ...turn.timestamps },
+    endpointCommitLatencyMs: turn.endpointCommitLatencyMs,
     endpointHoldCount: turn.endpointHoldCount,
     endpointDecisionMaxLatencyMs: turn.endpointDecisionMaxLatencyMs,
-    ackSpoken: turn.ackSpoken,
+    endpointDecisionSource: turn.endpointDecisionSource,
     progressUpdatesSpoken: turn.progressUpdatesSpoken,
   };
 }
@@ -663,7 +684,7 @@ function snapshotTurn(turn: MutableTurn): LiveVoiceTurnMetrics {
     status: turn.status,
     cancellationReason: turn.cancellationReason,
     timestamps,
-    ...frontModelFields(turn),
+    ...optionalTurnFields(turn),
     durations: {
       firstAudioToFirstPartialMs: duration(
         timestamps.firstAudioAtMs,
@@ -766,6 +787,10 @@ function percentile(
   return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)];
 }
 
+function normalizeLatencyMs(latencyMs: number): number {
+  return Number.isFinite(latencyMs) ? Math.max(0, latencyMs) : 0;
+}
+
 function duration(startMs: number | null, endMs: number | null): number | null {
   if (startMs === null || endMs === null) {
     return null;

package/src/live-voice/live-voice-photo.ts

@@ -0,0 +1,168 @@
+/**
+ * Persisting a photo taken during a live-voice call.
+ *
+ * A photo lands in the conversation as its own user message, the moment the
+ * shutter fires, and **runs no turn**. That single choice is what makes the
+ * order of shutter and speech irrelevant: whatever the user says next, before
+ * or after the snap, is answered by a model whose history already contains the
+ * image.
+ *
+ * The alternatives were both worse and both were tried. Attaching the photo to
+ * the *next* spoken turn strands it whenever the user speaks first: the
+ * question "what's this?" dispatches without an image and the photo waits for
+ * a sentence that has already been said. Dispatching a turn *for* the photo
+ * races the sentence the user is in the middle of saying, so the assistant
+ * answers the picture and the words separately.
+ *
+ * The cost, accepted: media survives a context-overflow retry only on the most
+ * recent user message (`conversation-media-retry.ts`), so on a long call a
+ * photo can be stripped where one riding the spoken turn would have survived.
+ * Stranding is the worse failure, and it happens every time rather than only
+ * under overflow.
+ */
+
+import { v7 as uuidv7 } from "uuid";
+
+import { persistQueuedMessageBody } from "../daemon/conversation-messaging.js";
+import { getOrCreateConversation } from "../daemon/conversation-store.js";
+import {
+  getAttachmentsByIds,
+  getSourcePathsForAttachments,
+} from "../persistence/attachments-store.js";
+import { recordConversationPersistedSeq } from "../persistence/conversation-crud.js";
+import { broadcastMessage } from "../runtime/assistant-event-hub.js";
+import { getCurrentSeq } from "../runtime/assistant-stream-state.js";
+import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
+import { getLogger } from "../util/logger.js";
+
+const log = getLogger("live-voice-photo");
+
+/**
+ * How long to wait for an in-flight turn before giving up on a photo.
+ *
+ * Persisting takes the conversation's processing lock, which a running turn
+ * holds for as long as it runs, tools included. The wait is generous because
+ * the alternative is dropping a photo the user watched themselves take, and
+ * the photo is not urgent: nothing is blocked on it except the next thing they
+ * say.
+ */
+const PROCESSING_WAIT_MS = 30_000;
+const PROCESSING_POLL_MS = 100;
+
+const PHOTO_MESSAGE_CONTENT = "here's a photo:";
+
+export interface LiveVoicePhotoResult {
+  readonly ok: boolean;
+  readonly messageId?: string;
+}
+
+/**
+ * Hydrate attachment ids into the shape `persistQueuedMessageBody` stores.
+ * Mirrors the HTTP send path's own `resolveAttachments`, so a photo taken
+ * mid-call is stored exactly like one attached to a typed message.
+ */
+function resolvePhotoAttachments(attachmentIds: string[]) {
+  const resolved = getAttachmentsByIds(attachmentIds, {
+    hydrateFileData: true,
+  });
+  const sourcePaths = getSourcePathsForAttachments(attachmentIds);
+  return resolved.map((a) => ({
+    id: a.id,
+    filename: a.originalFilename,
+    mimeType: a.mimeType,
+    data: a.dataBase64,
+    ...(sourcePaths.has(a.id) ? { filePath: sourcePaths.get(a.id) } : {}),
+  }));
+}
+
+/** Resolve once the conversation is not mid-turn, or false on timeout. */
+async function waitForIdle(conversation: {
+  isProcessing: () => boolean;
+}): Promise<boolean> {
+  const deadline = Date.now() + PROCESSING_WAIT_MS;
+  while (conversation.isProcessing()) {
+    if (Date.now() >= deadline) {
+      return false;
+    }
+    await new Promise((resolve) => setTimeout(resolve, PROCESSING_POLL_MS));
+  }
+  return true;
+}
+
+/**
+ * Persist a photo into the conversation as a user message, running no turn.
+ *
+ * Takes and releases the processing lock the way the built-in slash commands
+ * do: they are the existing precedent for "persist a user message and answer
+ * without the agent loop", and the lock is what keeps this write from
+ * interleaving with a turn's own persist.
+ *
+ * Never throws. A photo that cannot be stored must not take the call down with
+ * it, and the caller reports the failure to the user instead.
+ */
+export async function persistLiveVoicePhoto(
+  conversationId: string,
+  attachmentId: string,
+): Promise<LiveVoicePhotoResult> {
+  try {
+    const attachments = resolvePhotoAttachments([attachmentId]);
+    if (attachments.length === 0) {
+      log.warn({ attachmentId }, "Live-voice photo attachment did not resolve");
+      return { ok: false };
+    }
+
+    const conversation = await getOrCreateConversation(conversationId);
+
+    // A turn holds the lock for its whole run. Waiting rather than queueing:
+    // the conversation's queue drains into a turn, which is the one thing this
+    // must not cause.
+    if (!(await waitForIdle(conversation))) {
+      log.warn(
+        { conversationId, attachmentId },
+        "Live-voice photo timed out waiting for the conversation to go idle",
+      );
+      return { ok: false };
+    }
+
+    conversation.setProcessing(true);
+    try {
+      const persisted = await persistQueuedMessageBody(conversation, {
+        content: PHOTO_MESSAGE_CONTENT,
+        attachments,
+        requestId: uuidv7(),
+        metadata: {
+          // Marks the row as something the user did on a call rather than
+          // typed, the same way a voice turn's own user message is marked.
+          voiceSessionTurn: true,
+          livePhoto: true,
+        },
+      });
+
+      // The row exists but no turn will announce it, so the clients that
+      // render this conversation have to be told directly, or the photo does
+      // not appear until something else forces a refetch.
+      broadcastMessage({
+        type: "user_message_echo",
+        text: PHOTO_MESSAGE_CONTENT,
+        conversationId,
+        messageId: persisted.id,
+      });
+      recordConversationPersistedSeq(conversationId, getCurrentSeq());
+      publishConversationMessagesChanged(conversationId);
+
+      return { ok: true, messageId: persisted.id };
+    } finally {
+      conversation.setProcessing(false);
+      // Anything queued behind the lock we just held still has to run. Without
+      // this a message queued during the photo's write sits until the next
+      // turn ends.
+      void conversation.kickDrainQueue("loop_complete", "live_voice_photo");
+    }
+  } catch (err) {
+    log.warn(
+      { err, conversationId, attachmentId },
+      "Failed to persist a live-voice photo",
+    );
+    return { ok: false };
+  }
+}