@vellumai/assistant 0.11.6 → 0.11.7-staging.1

package/src/live-voice/live-voice-session-manager.ts

@@ -6,6 +6,7 @@ import {
   type LiveVoiceClientStartFrame,
   type LiveVoiceServerFrame,
   type LiveVoiceServerFramePayload,
+  type LiveVoiceSessionHolder,
 } from "./protocol.js";
 
 type MaybePromise<T> = T | Promise<T>;
@@ -157,6 +158,23 @@ function waitForSlotRelease(
   });
 }
 
+/** {@link LiveVoiceSessionHolder}, as the manager keeps it while filling it in. */
+interface MutableLiveVoiceSessionHolder {
+  client?: LiveVoiceSessionHolder["client"];
+  conversationId?: string;
+}
+
+/** Drop an all-empty holder, so `busy` carries a field or says nothing. */
+function describeHolder(
+  holder: MutableLiveVoiceSessionHolder,
+): LiveVoiceSessionHolder | null {
+  const described: MutableLiveVoiceSessionHolder = {
+    ...(holder.client ? { client: holder.client } : {}),
+    ...(holder.conversationId ? { conversationId: holder.conversationId } : {}),
+  };
+  return Object.keys(described).length > 0 ? described : null;
+}
+
 export class LiveVoiceSessionStartupError extends Error {
   constructor(message: string) {
     super(message);
@@ -208,6 +226,8 @@ interface ActiveLiveVoiceSession {
   sessionId: string;
   session: LiveVoiceSession;
   sink: LiveVoiceServerFrameSink;
+  /** What a refused client is told about who holds the slot. */
+  holder: MutableLiveVoiceSessionHolder;
   closing: boolean;
   silenceTimer: ReturnType<typeof setTimeout> | null;
   /** Resolves once this session has let go of the slot, however it went. */
@@ -272,12 +292,15 @@ export class LiveVoiceSessionManager {
       }
     }
 
-    const existingSessionId = this.activeSessionId;
-    if (existingSessionId !== null) {
+    const existing = this.activeSession;
+    const existingSessionId = existing?.sessionId ?? null;
+    if (existing !== null && existingSessionId !== null) {
       const busySequencer = createLiveVoiceServerFrameSequencer();
+      const holder = describeHolder(existing.holder);
       const frame = busySequencer.next({
         type: "busy",
         activeSessionId: existingSessionId,
+        ...(holder ? { holder } : {}),
       });
       await sink.sendFrame(frame);
       return {
@@ -289,10 +312,23 @@ export class LiveVoiceSessionManager {
 
     const sessionId = this.createSessionId();
     const sequencer = createLiveVoiceServerFrameSequencer();
+    const holder: MutableLiveVoiceSessionHolder = {
+      ...(startFrame.client ? { client: startFrame.client } : {}),
+      ...(startFrame.conversationId
+        ? { conversationId: startFrame.conversationId }
+        : {}),
+    };
     const context: LiveVoiceSessionFactoryContext = {
       sessionId,
       startFrame,
       sendFrame: async (payload) => {
+        // `ready` carries the conversation the session actually landed in,
+        // which is the only source for one it minted itself. Read in passing
+        // rather than through a new session method: every frame already goes
+        // through here.
+        if (payload.type === "ready") {
+          holder.conversationId = payload.conversationId;
+        }
         const frame = sequencer.next(payload);
         await sink.sendFrame(frame);
         return frame;
@@ -310,6 +346,7 @@ export class LiveVoiceSessionManager {
       sessionId,
       session,
       sink,
+      holder,
       closing: false,
       silenceTimer: null,
       released,

package/src/live-voice/live-voice-session.ts

@@ -59,6 +59,7 @@ import {
   dominantLanguageTag,
   voteDominantLanguage,
 } from "../stt/language-metadata.js";
+import { sttCatalogKeyForRole } from "../stt/roles.js";
 import {
   DEFAULT_SPEECH_ENERGY_THRESHOLD,
   pcm16MaxNormalizedCorrelation,
@@ -131,6 +132,7 @@ import {
 import {
   type LiveVoiceClientAttachImageFrame,
   type LiveVoiceClientFrame,
+  type LiveVoiceClientTextTurnFrame,
   type LiveVoiceClientUpdateConfigFrame,
   LiveVoiceProtocolErrorCode,
   type LiveVoiceServerFramePayload,
@@ -360,7 +362,8 @@ export interface LiveVoiceSessionOptions {
   /**
    * Deepgram Flux turn-detection tuning. The production factory seeds this
    * from `liveVoice.flux` config when unset; absent fields fall back to the
-   * schema defaults, which leave `turnEnd.enabled` false.
+   * schema defaults, which leave `turnEnd.enabled` on. A test that needs the
+   * local silence boundary has to say so.
    */
   fluxConfig?: Partial<LiveVoiceFluxConfig>;
   /**
@@ -705,6 +708,10 @@ interface ActiveAssistantTurn {
   // no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
   // the answer rides the control prompt (buildLiveDeliveryNote).
   continuationDelivery: ContinuationDelivery | null;
+  // The turn's content is an internal instruction rather than user speech (the
+  // greeting that opens a session, say). The row still persists and the model
+  // still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
+  hiddenPrompt: boolean;
   // Set when a barge-in handed the interrupted work to a background subagent:
   // that request's transcript, so the model can tell the user the work is
   // still running instead of appearing to have dropped it.
@@ -993,6 +1000,19 @@ function createSyntheticUtterance(): UtteranceCycle {
   };
 }
 
+// Carrier cycle for a turn the user typed rather than spoke. Identical to the
+// announcement carrier except that it arrives with its transcript already
+// complete: the text the client sent is the finalized transcript, so the cycle
+// joins the pipeline exactly where STT would have handed one over. Like the
+// announcement carrier it is never installed as `currentUtterance`, so a typed
+// turn taken during a voice session cannot collide with the armed capture cycle.
+function createTypedUtterance(text: string): UtteranceCycle {
+  return {
+    ...createSyntheticUtterance(),
+    finalTranscriptSegments: [text],
+  };
+}
+
 // Objective handed to the background subagent that continues a barged-in turn.
 // The subagent forks the live conversation, so it already sees the interrupted
 // turn's completed tool calls in history and resumes from there. The request
@@ -1143,6 +1163,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   private receivedAudio = false;
   private detectedSpeech = false;
   private dispatchedTurn = false;
+  // The client declared a text input affordance on the start frame, so it can
+  // take a turn without the microphone. Governs one thing only: whether a
+  // missing speech-to-text leg is fatal to startup (see start()).
+  private readonly textInput: boolean;
+  // Whether this session's speech-to-text leg came up. False only when the
+  // preflight found it missing and `textInput` let the session open anyway, in
+  // which case nothing arms a transcriber and typed turns are the only input.
+  private audioInput = true;
   // Non-null iff the start frame requested turnDetection "server_vad".
   private readonly turnDetector: MediaTurnDetector | null;
   // Base energy gate for server-VAD speech classification. During estimated
@@ -1398,6 +1426,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       context.startFrame.audio.sampleRate *
       2 *
       SERVER_VAD_PENDING_AUDIO_MAX_SECONDS;
+    this.textInput = context.startFrame.textInput === true;
   }
 
   get finalTranscriptText(): string {
@@ -1418,10 +1447,30 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     if (this.resolveCredentialReadiness) {
       const readiness = await this.resolveCredentialReadiness();
       if (readiness.status === "not-ready") {
-        return await this.failStartup(
-          readiness.userMessage,
-          LiveVoiceProtocolErrorCode.CredentialsUnavailable,
-        );
+        // A client that can type survives a dead speech-to-text leg: it opens
+        // text-only rather than not at all. Requires the gap to be entirely on
+        // the STT side, because the text-in path still speaks its reply, so a
+        // session missing the TTS leg has nothing left to degrade to.
+        //
+        // A not-ready verdict naming no gap is not a downgrade candidate:
+        // `every` is vacuously true over an empty list, and a resolver bug must
+        // not read as "only STT is missing" and quietly open a half-working
+        // session on the strength of it.
+        const sttOnlyGap =
+          readiness.missing.length > 0 &&
+          readiness.missing.every((gap) => gap.kind === "stt");
+        if (this.textInput && sttOnlyGap) {
+          this.audioInput = false;
+          log.warn(
+            { sessionId: this.context.sessionId, missing: readiness.missing },
+            "Live voice starting text-only: speech-to-text unavailable",
+          );
+        } else {
+          return await this.failStartup(
+            readiness.userMessage,
+            LiveVoiceProtocolErrorCode.CredentialsUnavailable,
+          );
+        }
       }
     }
 
@@ -1444,6 +1493,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       sessionId: this.context.sessionId,
       conversationId: this.conversationId,
       turnDetection: this.turnDetector ? "server_vad" : "manual",
+      // Unconditional: it answers "does this daemon take text frames", which
+      // is a property of the daemon and not of what this client asked for.
+      // A client reading it back absent is talking to one that predates it.
+      textInput: true,
+      ...(this.audioInput ? {} : { audioInput: false }),
     });
   }
 
@@ -1472,9 +1526,66 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       case "attach_image":
         this.persistPhoto(frame);
         return;
+      case "text":
+        await this.handleTextTurn(frame);
+        return;
     }
   }
 
+  /**
+   * Run a turn the user typed instead of spoke.
+   *
+   * Joins the pipeline at the point a finished transcript would have: the same
+   * turn runner, the same streaming TTS, the same activity and metrics. The
+   * reply is spoken, which is the whole point: this is the input half being
+   * text, not the session becoming a chat.
+   *
+   * Gated on {@link sessionTurnFloorBlocker}, the same check the continuation
+   * announcement uses: no turn in flight, no assistant audio still draining,
+   * no speech onset, and no utterance mid-capture. A typed turn asks the
+   * identical question, so it asks it in the identical way rather than growing
+   * a second set of conditions that could drift from the first.
+   *
+   * A blocked turn is refused rather than queued, and refused recoverably: the
+   * session is fine and the user can send again. Typing over a reply in
+   * progress reads as an interrupt, but making it one means choosing barge-in
+   * semantics for a modality that has no speech onset to measure, which is its
+   * own decision and not this seam's.
+   */
+  private async handleTextTurn(
+    frame: LiveVoiceClientTextTurnFrame,
+  ): Promise<void> {
+    if (!this.startVoiceTurn) {
+      return;
+    }
+
+    const blocker = this.sessionTurnFloorBlocker({
+      ignoreManualCapture: true,
+    });
+    if (blocker !== null) {
+      await this.sendFrame({
+        type: "error",
+        code: LiveVoiceProtocolErrorCode.InvalidFrame,
+        message: "The assistant is busy with the current turn. Send again.",
+        frameType: "text",
+        recoverable: true,
+      });
+      log.debug(
+        { sessionId: this.context.sessionId, blocker },
+        "Typed live-voice turn refused",
+      );
+      return;
+    }
+
+    const utterance = createTypedUtterance(frame.text);
+    // Before the launch, so the anchor is the moment the text arrived rather
+    // than whatever the dispatch costs.
+    this.markTypedTurnSubmitted(utterance);
+    await this.launchAssistantTurn(utterance, frame.text, {
+      ...(frame.hidden === true ? { hiddenPrompt: true } : {}),
+    });
+  }
+
   /**
    * Persist a photo taken mid-call into the conversation, running no turn.
    *
@@ -1546,6 +1657,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       ? null
       : liveVoiceSilenceReason({
           reachedActive: this.reachedActive,
+          audioInput: this.audioInput,
           receivedAudio: this.receivedAudio,
           detectedSpeech: this.detectedSpeech,
         });
@@ -1637,9 +1749,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       // outside: the session looks like a turn-detecting session while its
       // opening turn is not one. The resolved provider reconciles this guess
       // below.
+      //
+      // The guess reads the live-voice role, which is what the dial below
+      // resolves. Reading the global provider instead makes the guess wrong
+      // in exactly the configuration roles exist for (live voice on flux,
+      // global on base deepgram), which reinstates the race this seed
+      // prevents.
       this.setProviderTurnEndActive(
         this.fluxConfig.turnEnd.enabled &&
-          supportsProviderTurnDetection(stt.provider as SttProviderId) &&
+          supportsProviderTurnDetection(
+            sttCatalogKeyForRole(stt, "liveVoice"),
+          ) &&
           this.turnDetector !== null,
       );
       const transcriber = await this.resolveTranscriber({
@@ -1838,6 +1958,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   }
 
   private async armUtterance(): Promise<void> {
+    // Nothing to arm without a speech-to-text leg. Guarded here rather than at
+    // the call sites because there are several, and the post-turn re-arm
+    // reaches this through `scheduleRearmAfterTurn` from five of them: a
+    // text-only session that requested `server_vad` still holds a turn
+    // detector, so the re-arm is not skipped for want of one. Arming anyway
+    // would resolve a transcriber the preflight already said has no
+    // credential, fail the session, and close it after its first typed turn.
+    if (!this.audioInput) {
+      return;
+    }
+
     const result = await this.beginUtterance();
     if (result.status === "started" || result.status === "stale") {
       return;
@@ -1867,6 +1998,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     // open", which a chunk arriving at all settles regardless of what the
     // session then does with it.
     this.receivedAudio = true;
+    // A text-only session armed no transcriber, so there is nothing downstream
+    // to route this into. Dropped rather than errored: the client was told on
+    // `ready` that the microphone leg is dead, and a client still streaming is
+    // one that has not caught up, not one to end the session over. The flag
+    // above is still set, so telemetry keeps seeing that the mic did open.
+    if (!this.audioInput) {
+      return;
+    }
     if (this.turnDetector) {
       await this.handleServerVadAudio(this.turnDetector, chunk);
       return;
@@ -3118,6 +3257,41 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     if (this.pendingAnnouncement === null) {
       return "nothing_pending";
     }
+    return this.sessionTurnFloorBlocker();
+  }
+
+  /**
+   * Why the session may not start a turn of its own right now, or null when it
+   * may.
+   *
+   * "Of its own" means a turn with no released utterance behind it: a queued
+   * continuation announcement, or a turn the user typed. Both need the same
+   * assurance, that starting one would not talk over the assistant, cut off the
+   * user, or race a capture cycle already in flight, so both ask here rather
+   * than growing separate condition sets that could drift apart.
+   */
+  private sessionTurnFloorBlocker(opts?: {
+    /**
+     * Ignore `manualAudioCaptured` when judging whether an utterance is in
+     * flight. Set only for a typed turn.
+     *
+     * That flag means "manual capture routed at least one chunk into this
+     * cycle", and manual capture forwards from the moment the microphone
+     * opens rather than from a push-to-talk press. So in a manual session it
+     * latches on the first chunk of silence and, because a manual cycle only
+     * completes on release, never clears. Honouring it would refuse every
+     * typed turn a few milliseconds into the session.
+     *
+     * Safe to ignore here and nowhere else. The flag exists because
+     * transcript signals trail the provider, which would leave an
+     * announcement talking over someone who is mid-sentence with no text yet.
+     * A typed turn has evidence an announcement does not: the user is at the
+     * keyboard, and the act of sending is itself the signal that they are not
+     * currently speaking. The transcript-derived conditions still apply, so a
+     * cycle that has actually produced words still blocks.
+     */
+    readonly ignoreManualCapture?: boolean;
+  }): string | null {
     if (this.isClosed || this.state === "failed" || !this.startVoiceTurn) {
       return "session_unavailable";
     }
@@ -3146,7 +3320,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       !utterance.completed &&
       (utterance.released ||
         utterance.assistantTurnStarted ||
-        utterance.manualAudioCaptured ||
+        (utterance.manualAudioCaptured && opts?.ignoreManualCapture !== true) ||
         utterance.finalTranscriptSegments.length > 0 ||
         utterance.latestPartialText !== null)
     ) {
@@ -4544,6 +4718,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       // leading verdict commits the turn (see commitSpeculativeTurn); a hold
       // verdict rolls everything back instead.
       speculative?: boolean;
+      // Marks `content` as an internal instruction rather than user speech, so
+      // the row persists and drives the turn but never renders: no echo, and
+      // `/messages` filters it after a reload.
+      hiddenPrompt?: boolean;
     },
   ): Promise<boolean> {
     utterance.assistantTurnStarted = true;
@@ -4604,6 +4782,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       consumedAnnouncement: pending?.announcement ?? null,
       pendingContextStopGeneration: this.detachStopGeneration,
       continuationDelivery: opts?.continuationDelivery ?? null,
+      hiddenPrompt: opts?.hiddenPrompt === true,
       deltaEpoch: 0,
       escalationHandedOff: false,
       ttsBuffer: "",
@@ -4778,6 +4957,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       const handle = await this.startVoiceTurn({
         conversationId: this.conversationId,
         voiceSessionId: this.context.sessionId,
+        ...(activeTurn.hiddenPrompt ? { hiddenSyntheticPrompt: true } : {}),
         userMessageChannel: "vellum",
         assistantMessageChannel: "vellum",
         // Fixed, and NOT the originating client: this pair resolves the turn's
@@ -5950,6 +6130,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     );
   }
 
+  /**
+   * Stamp the moment a typed turn's text arrived as the end of the user's
+   * input.
+   *
+   * `roundTripMs` measures from the user finishing their turn to the first
+   * audio they hear, and it anchors on `utteranceEndAtMs ?? pttReleaseAtMs`.
+   * A typed turn produces neither, so without this every typed turn reports a
+   * null round trip and disappears from the latency numbers. That would bias
+   * them toward spoken turns rather than simply losing rows, which is the
+   * worse failure: the metric would still look healthy while describing less
+   * and less of the traffic.
+   *
+   * The text frame landing is the honest equivalent of an utterance closing:
+   * it is the instant the user stopped providing input and started waiting.
+   * `sttMs` and the partial-transcript durations stay null, because nothing
+   * was transcribed and claiming otherwise would invent a measurement.
+   */
+  private markTypedTurnSubmitted(utterance: UtteranceCycle): void {
+    this.markUtteranceMetric(utterance, "utteranceEndAtMs", (turnId) =>
+      this.metrics.markUtteranceEnd(turnId),
+    );
+  }
+
   // Manual mode stamps the PTT release; server_vad stamps the utterance-end
   // boundary instead (utteranceEndToFinalTranscript plays the sttMs role).
   private markUtteranceReleased(utterance: UtteranceCycle): void {
@@ -6609,7 +6812,9 @@ async function defaultResolveStreamingTranscriber(
     await import("../config/managed-speech-defaults.js");
   const { resolveStreamingTranscriber } =
     await import("../providers/speech-to-text/resolve.js");
-  const { stt } = await resolveEffectiveSpeechProviders();
+  const { stt } = await resolveEffectiveSpeechProviders(undefined, {
+    role: "liveVoice",
+  });
   return resolveStreamingTranscriber({ ...options, providerId: stt });
 }