@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0

package/src/calls/call-transport.ts

@@ -98,4 +98,20 @@ export interface CallTransport {
    * this — the controller then treats playback as already drained.
    */
   awaitPlaybackDrained?(): Promise<void>;
+
+  /**
+   * Whether no assistant speech is buffered, queued, in flight, or
+   * (estimatedly) still playing to the caller. Progress narration speaks
+   * only into audible silence, so it asks before every update. Transports
+   * that play speech synchronously may omit this; the controller then
+   * treats the transport as idle.
+   */
+  isPlaybackIdle?(): boolean;
+
+  /**
+   * Wall-clock estimate of when audio already sent finishes playing to the
+   * caller, 0 when nothing is playing. Anchors the dead-air countdown for
+   * progress narration to when the caller last heard something.
+   */
+  playbackTailUntilMs?(): number;
 }

package/src/calls/media-stream-output.ts

@@ -327,6 +327,14 @@ export class MediaStreamOutput implements CallTransport {
   /** True when the queue drain loop is actively running. */
   private draining = false;
 
+  /**
+   * Estimated wall-clock instant the audio already sent to Twilio finishes
+   * playing to the caller. Frames leave faster than real time and Twilio
+   * buffers them, so the only local view of "still audible" is this running
+   * sum of sent frame durations; a `clear` drops the buffer and resets it.
+   */
+  private playbackTailAtMs = 0;
+
   /** Abort controller for the currently in-flight synthesis/fetch. */
   private activePlaybackAbort: AbortController | null = null;
 
@@ -659,6 +667,7 @@ export class MediaStreamOutput implements CallTransport {
       return;
     }
     this.sendClearCommand();
+    this.playbackTailAtMs = 0;
     // Twilio drops its buffered audio (and the marks within it) on `clear`, so
     // any end-of-turn mark already sent will never echo. Treat those as drained
     // so a pending end-call drain wait resolves instead of stalling on the cap.
@@ -716,6 +725,24 @@ export class MediaStreamOutput implements CallTransport {
     return this.playbackQueue.length;
   }
 
+  /**
+   * Whether no speech is buffered toward a sentence, queued, being
+   * synthesized or fetched, or (estimatedly) still playing at Twilio.
+   */
+  isPlaybackIdle(): boolean {
+    return (
+      this.textBuffer.length === 0 &&
+      this.playbackQueue.length === 0 &&
+      !this.draining &&
+      Date.now() >= this.playbackTailAtMs
+    );
+  }
+
+  /** See {@link CallTransport.playbackTailUntilMs}. */
+  playbackTailUntilMs(): number {
+    return this.playbackTailAtMs;
+  }
+
   /**
    * Runtime check for closed state. Used instead of direct property access
    * in async methods because TypeScript's control flow analysis cannot
@@ -772,6 +799,7 @@ export class MediaStreamOutput implements CallTransport {
     this.playbackQueue.length = 0;
     this.playbackVersion++;
     this.audioStartCallback = null;
+    this.playbackTailAtMs = 0;
     if (this.activePlaybackAbort) {
       this.activePlaybackAbort.abort();
       this.activePlaybackAbort = null;
@@ -854,6 +882,10 @@ export class MediaStreamOutput implements CallTransport {
     for (const frame of frames) {
       this.sendAudioPayload(frame);
     }
+    const durationMs =
+      (frames.length * MULAW_FRAME_SIZE * 1000) / TELEPHONY_SAMPLE_RATE_HZ;
+    this.playbackTailAtMs =
+      Math.max(Date.now(), this.playbackTailAtMs) + durationMs;
   }
 
   /**

package/src/calls/media-stream-server.ts

@@ -47,6 +47,11 @@ import { addMessage } from "../persistence/conversation-crud.js";
 import { resolveGuardianName } from "../prompts/user-reference.js";
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
 import { getLogger } from "../util/logger.js";
+import {
+  type BargeInGuard,
+  createBargeInGuard,
+  DEFAULT_BARGE_IN_MIN_SPEECH_MS,
+} from "./barge-in-guard.js";
 import { CallController } from "./call-controller.js";
 import {
   formatDuration,
@@ -188,6 +193,8 @@ export class MediaStreamCallSession {
   private bargeInAccepted = 0;
   /** Number of barge-in attempts that were ignored (assistant not speaking). */
   private bargeInIgnored = 0;
+  /** Armed by caller speech while a turn or its tail is interruptible. */
+  private bargeInGuard: BargeInGuard | null = null;
   /** Number of turn-start transitions detected by the STT session. */
   private turnStarts = 0;
   /** Number of transcript finals produced (non-empty). */
@@ -215,6 +222,9 @@ export class MediaStreamCallSession {
     // Create STT session with callbacks wired to the controller.
     const callbacks: MediaStreamSttSessionCallbacks = {
       onSpeechStart: () => this.handleSpeechStart(),
+      onMediaFrame: (hasSpeech, durationMs) =>
+        this.handleMediaFrame(hasSpeech, durationMs),
+      onSpeechEnd: () => this.handleSpeechEnd(),
       onTranscriptFinal: (text, durationMs) =>
         this.handleTranscriptFinal(text, durationMs),
       onDtmf: (digit) => this.handleDtmf(digit),
@@ -768,38 +778,80 @@ export class MediaStreamCallSession {
 
   private handleSpeechStart(): void {
     this.turnStarts++;
+  }
 
-    // Barge-in: clear queued outbound audio and abort the in-flight LLM
-    // turn only when the assistant is actively speaking. Uses the gated
-    // handleBargeIn path so initial inbound audio frames do not cancel a
-    // still-starting initial turn.
-    //
-    // clearAudio runs via the onAccepted hook so it only fires when the
-    // barge-in passes the speaking gate — an ignored barge-in (controller
-    // idle/processing) must not flush queued/in-flight TTS such as a
-    // buffered greeting. The hook runs before handleInterrupt so the
-    // end-of-turn mark it enqueues survives the queue flush.
-    if (this.output && this.controller) {
-      const output = this.output;
-      const accepted = this.controller.handleBargeIn(() => output.clearAudio());
-      if (accepted) {
-        this.bargeInAccepted++;
-        log.info(
-          { callSessionId: this.callSessionId },
-          "Media-stream barge-in accepted — cleared outbound audio",
-        );
-      } else {
-        // No turn to abort, but a completed turn's tail can still be
-        // playing from Twilio's buffer — flush only that buffer so the
-        // caller isn't talked over. Queued speech that hasn't reached
-        // Twilio yet (greeting, handoff prompt) is preserved.
-        output.clearBufferedAudio();
-        this.bargeInIgnored++;
-        log.debug(
-          { callSessionId: this.callSessionId },
-          "Media-stream barge-in ignored — assistant not speaking",
-        );
+  // Sustained-speech barge-in guard, fed by every inbound frame. Speech only
+  // counts while there is something to interrupt: an assistant turn in
+  // flight (thinking or speaking) or a completed turn's tail still playing
+  // from Twilio's buffer. Outside that window the guard is dropped, so
+  // speech from the caller's own utterance never carries into a turn that
+  // starts before the local VAD has ended the utterance (a streaming final
+  // can start the turn first). A speech frame inside the window arms the
+  // guard; a fired guard reaches the controller. A line click, a cough, or
+  // TTS bleed never gets that far.
+  private handleMediaFrame(hasSpeech: boolean, durationMs: number): void {
+    if (!this.output || !this.controller) {
+      return;
+    }
+    const bargeable =
+      this.controller.getState() !== "idle" || !this.output.isPlaybackIdle();
+    if (!bargeable) {
+      this.bargeInGuard = null;
+      return;
+    }
+    if (this.bargeInGuard === null) {
+      if (!hasSpeech) {
+        return;
       }
+      this.bargeInGuard = createBargeInGuard(DEFAULT_BARGE_IN_MIN_SPEECH_MS);
+    }
+    const step = this.bargeInGuard.track(
+      hasSpeech ? "speech" : "silence",
+      durationMs,
+    );
+    if (step === "fired") {
+      this.bargeInGuard = null;
+      this.fireBargeIn();
+    }
+  }
+
+  private handleSpeechEnd(): void {
+    // An untripped guard was noise or a fragment: the run never reached the
+    // threshold before the caller's turn ended.
+    this.bargeInGuard = null;
+  }
+
+  // Sustained caller speech landed: clear queued outbound audio and abort
+  // the in-flight LLM turn when one is in flight (thinking or speaking).
+  //
+  // clearAudio runs via the onAccepted hook so it only fires when the
+  // barge-in passes the controller's gate; an ignored barge-in (controller
+  // idle) must not flush queued/in-flight TTS such as a buffered greeting.
+  // The hook runs before handleInterrupt so the end-of-turn mark it
+  // enqueues survives the queue flush.
+  private fireBargeIn(): void {
+    if (!this.output || !this.controller) {
+      return;
+    }
+    const output = this.output;
+    const accepted = this.controller.handleBargeIn(() => output.clearAudio());
+    if (accepted) {
+      this.bargeInAccepted++;
+      log.info(
+        { callSessionId: this.callSessionId },
+        "Media-stream barge-in accepted: cleared outbound audio",
+      );
+    } else {
+      // No turn to abort, but a completed turn's tail can still be
+      // playing from Twilio's buffer, so flush only that buffer so the
+      // caller isn't talked over. Queued speech that hasn't reached
+      // Twilio yet (greeting, handoff prompt) is preserved.
+      output.clearBufferedAudio();
+      this.bargeInIgnored++;
+      log.debug(
+        { callSessionId: this.callSessionId },
+        "Media-stream barge-in ignored: no turn in flight",
+      );
     }
   }
 

package/src/calls/media-stream-stt-session.ts

@@ -120,6 +120,16 @@ export interface MediaStreamSttSessionCallbacks {
   /** Called when the turn detector transitions to active (first speech-bearing chunk). */
   onSpeechStart?: () => void;
 
+  /**
+   * Called for every inbound frame with the local VAD's verdict on it and
+   * the frame's duration. Drives the sustained-speech barge-in guard, which
+   * needs the silence frames inside a detector turn as well as the speech.
+   */
+  onMediaFrame?: (hasSpeech: boolean, durationMs: number) => void;
+
+  /** Called when the turn detector ends the caller's turn (silence or cap). */
+  onSpeechEnd?: () => void;
+
   /**
    * Called when a completed caller utterance has been transcribed.
    *
@@ -255,6 +265,7 @@ export class MediaStreamSttSession {
         this.callbacks.onSpeechStart?.();
       },
       onTurnEnd: (reason, durationMs) => {
+        this.callbacks.onSpeechEnd?.();
         void this.handleTurnEnd(reason, durationMs);
       },
     });
@@ -377,6 +388,10 @@ export class MediaStreamSttSession {
     const raw = Buffer.from(event.media.payload, "base64");
     const hasSpeech = detectSpeechActivity(raw);
     this.turnDetector.onMediaChunk(hasSpeech);
+    this.callbacks.onMediaFrame?.(
+      hasSpeech,
+      raw.length / (TELEPHONY_SOURCE_SAMPLE_RATE / 1000),
+    );
 
     if (this.mode === "batch") {
       this.currentTurnChunks.push(event.media.payload);

package/src/{live-voice → calls}/progress-narration.ts

@@ -5,7 +5,7 @@
  * The voice front door owns hold, escalation, and direct-response behavior.
  */
 
-import type { LiveVoiceProgressConfig } from "../config/schemas/live-voice.js";
+import type { VoiceProgressConfig } from "../config/schemas/voice.js";
 import {
   extractToolUse,
   getConfiguredProvider,
@@ -16,7 +16,7 @@ import type {
   ProviderResponse,
   ToolDefinition,
 } from "../providers/types.js";
-import { createAbortReason } from "../util/abort-reasons.js";
+import { createAbortReason, isAbortReason } from "../util/abort-reasons.js";
 import { getLogger } from "../util/logger.js";
 
 const log = getLogger("voice-progress-narration");
@@ -164,7 +164,10 @@ async function requestBoundedResponse(args: {
   const timeoutTimer = setTimeout(
     () =>
       timeoutController.abort(
-        createAbortReason("voice_session_aborted", "voice-progress-narration"),
+        createAbortReason(
+          "voice_progress_narration_timeout",
+          "voice-progress-narration",
+        ),
       ),
     args.timeoutMs,
   );
@@ -216,7 +219,7 @@ export function effectiveSpokenTextMaxChars(
 }
 
 export function createVoiceProgressNarrator(options: {
-  config: LiveVoiceProgressConfig;
+  config: VoiceProgressConfig;
   /** Provider resolver, injectable for tests. */
   getProvider?: () => Promise<Provider | null>;
 }): VoiceProgressNarrator {
@@ -260,9 +263,19 @@ export function createVoiceProgressNarrator(options: {
         }
         return trimmed;
       } catch (error) {
+        // Why the beat went unspoken, as one greppable field. A narrator whose
+        // budget is below the model's real roundtrip drops every update at the
+        // deadline and reads in the logs as a run of unrelated failures; naming
+        // the outcome is what makes that run legible as "the feature is off".
+        const outcome = isAbortReason(error)
+          ? error.kind === "voice_progress_narration_timeout"
+            ? "timed_out"
+            : "session_ended"
+          : "error";
         log.info(
           {
             error,
+            outcome,
             providerResolveMs,
             totalMs: Math.round(performance.now() - startedAt),
             timeoutMs: options.config.generationTimeoutMs,

package/src/calls/resolve-call-tts-provider.ts

@@ -8,6 +8,7 @@
  */
 
 import { loadConfig } from "../config/loader.js";
+import { resolveEffectiveSpeechProviders } from "../config/managed-speech-defaults.js";
 import {
   getCatalogProvider,
   getTtsProvider,
@@ -73,6 +74,13 @@ export interface ResolveCallTtsOptions {
 /**
  * Resolve the active TTS provider via the global provider abstraction.
  *
+ * The provider is the effective one after managed-speech defaulting
+ * ({@link resolveEffectiveSpeechProviders}): a configured BYOK provider
+ * whose credential does not resolve is stood in for by managed speech
+ * while the platform connection is usable, the same substitution live
+ * voice makes, so a fresh Vellum connection speaks on the phone with zero
+ * TTS configuration.
+ *
  * The native-vs-synthesized decision is driven by the catalog's
  * `callMode` field via {@link resolveCallStrategy}. Providers with
  * `callMode: "synthesized-play"` have their audio streamed through the
@@ -94,10 +102,11 @@ export async function resolveCallTtsProvider(
 ): Promise<ResolvedCallTts> {
   try {
     const config = loadConfig();
-    const resolved = resolveTtsConfig(config);
+    const effectiveProviderId = await effectiveTtsProviderId(config);
+    const resolved = resolveTtsConfig(config, effectiveProviderId);
 
     // Use the catalog's callMode to decide the call path.
-    const strategy = resolveCallStrategy(config);
+    const strategy = resolveCallStrategy(config, effectiveProviderId);
 
     let providerId = resolved.provider;
     let useSynthesizedPath = strategy.callMode === "synthesized-play";
@@ -175,6 +184,28 @@ export async function resolveCallTtsProvider(
   }
 }
 
+/**
+ * The TTS provider the call actually uses after managed-speech defaulting.
+ * A failure inside the managed lookup (platform client, credential store)
+ * keeps the configured provider: a managed-speech hiccup must not demote a
+ * working BYOK provider, and the outer resolver's own fallback is the
+ * silent native path.
+ */
+async function effectiveTtsProviderId(
+  config: ReturnType<typeof loadConfig>,
+): Promise<TtsProviderId | undefined> {
+  try {
+    const { tts } = await resolveEffectiveSpeechProviders(config);
+    return tts;
+  } catch (err) {
+    log.warn(
+      { err },
+      "Managed-speech defaulting failed for call TTS; keeping the configured provider",
+    );
+    return undefined;
+  }
+}
+
 /**
  * Decode a resolved call audio format into the provider request format
  * and the audio-store format.

package/src/calls/tts-call-strategy.ts

@@ -40,17 +40,21 @@ export interface TtsCallStrategy {
 }
 
 /**
- * Resolve the call strategy for the currently configured TTS provider.
+ * Resolve the call strategy for the active TTS provider.
  *
- * Reads the active provider from config via {@link resolveTtsConfig},
- * then looks up the provider's `callMode` in the catalog.
+ * Reads the provider from config via {@link resolveTtsConfig} (or takes
+ * `providerOverride`, the provider a caller already resolved after
+ * managed-speech defaulting), then looks up its `callMode` in the catalog.
  *
  * Falls back to `native-twilio` with `"elevenlabs"` when the config
  * or catalog is unavailable (e.g. test mocks, pre-migration configs).
  */
-export function resolveCallStrategy(config: AssistantConfig): TtsCallStrategy {
+export function resolveCallStrategy(
+  config: AssistantConfig,
+  providerOverride?: TtsProviderId,
+): TtsCallStrategy {
   try {
-    const resolved = resolveTtsConfig(config);
+    const resolved = resolveTtsConfig(config, providerOverride);
     const catalogEntry = getCatalogProvider(resolved.provider);
     return {
       providerId: catalogEntry.id,

package/src/calls/types.ts

@@ -11,6 +11,8 @@ export type CallEventType =
   | "call_connected"
   | "caller_spoke"
   | "assistant_spoke"
+  | "tool_use_started"
+  | "tool_use_completed"
   | "user_question_asked"
   | "user_answered"
   | "user_instruction_relayed"