@vellumai/assistant 0.6.3 → 0.6.4

package/src/bundler/package-resolver.ts

@@ -33,6 +33,10 @@ const inflight = new Map<string, Promise<string | null>>();
 
 /** Where all cached packages live on disk. */
 export function getCacheDir(): string {
+  // Package cache is intentionally shared across all local assistants to
+  // avoid re-downloading the same bundler deps per instance. Resolved
+  // against the real home directory so the cache is a single host-wide
+  // location regardless of BASE_DATA_DIR / per-instance containers.
   return join(homedir(), ".vellum", "package-cache");
 }
 

package/src/calls/audio-store.ts

@@ -24,7 +24,7 @@ let currentBytes = 0;
 
 export function storeAudio(
   buffer: Buffer,
-  format: "mp3" | "wav" | "opus",
+  format: "mp3" | "wav" | "opus" | "pcm",
 ): string {
   evictExpired();
   // Evict oldest if over capacity
@@ -50,7 +50,7 @@ export interface StreamingAudioHandle {
 }
 
 export function createStreamingEntry(
-  format: "mp3" | "wav" | "opus",
+  format: "mp3" | "wav" | "opus" | "pcm",
 ): StreamingAudioHandle {
   evictExpired();
   const id = randomUUID();
@@ -151,19 +151,25 @@ export function getAudio(id: string): AudioResult | null {
     removeEntry(id);
     return null;
   }
-  return { type: "buffer", buffer: entry.buffer, contentType: entry.contentType };
+  return {
+    type: "buffer",
+    buffer: entry.buffer,
+    contentType: entry.contentType,
+  };
 }
 
 // ---------------------------------------------------------------------------
 // Internal helpers
 // ---------------------------------------------------------------------------
 
-function contentTypeForFormat(format: "mp3" | "wav" | "opus"): string {
+function contentTypeForFormat(format: "mp3" | "wav" | "opus" | "pcm"): string {
   return format === "mp3"
     ? "audio/mpeg"
     : format === "wav"
       ? "audio/wav"
-      : "audio/opus";
+      : format === "pcm"
+        ? "audio/pcm"
+        : "audio/opus";
 }
 
 function mergeChunks(chunks: Uint8Array[]): Uint8Array {

package/src/calls/call-controller.ts

@@ -23,6 +23,8 @@ import { revokeScopedApprovalGrantsForContext } from "../memory/scoped-approval-
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
 import { mintDaemonDeliveryToken } from "../runtime/auth/token-service.js";
 import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
+import { getCatalogProvider } from "../tts/provider-catalog.js";
+import type { TtsProvider, TtsProviderId } from "../tts/types.js";
 import { getLogger } from "../util/logger.js";
 import { createStreamingEntry } from "./audio-store.js";
 import {
@@ -44,11 +46,11 @@ import {
   recordCallEvent,
   updateCallSession,
 } from "./call-store.js";
+import type { CallTransport } from "./call-transport.js";
 import { finalizeCall } from "./finalize-call.js";
-import { synthesizeWithFishAudio } from "./fish-audio-client.js";
 import { sendGuardianExpiryNotices } from "./guardian-action-sweep.js";
 import { dispatchGuardianQuestion } from "./guardian-dispatch.js";
-import type { RelayConnection } from "./relay-server.js";
+import { resolveCallTtsProvider } from "./resolve-call-tts-provider.js";
 import type { PromptSpeakerContext } from "./speaker-identification.js";
 import { sanitizeForTts } from "./tts-text-sanitizer.js";
 import {
@@ -61,7 +63,6 @@ import {
   extractBalancedJson,
   stripInternalSpeechMarkers,
 } from "./voice-control-protocol.js";
-import { isFishAudioTts } from "./voice-quality.js";
 import {
   startVoiceTurn,
   type VoiceTurnHandle,
@@ -87,7 +88,7 @@ interface PendingGuardianInput {
 
 export class CallController {
   private callSessionId: string;
-  private relay: RelayConnection;
+  private transport: CallTransport;
   private state: ControllerState = "idle";
   private abortController: AbortController = new AbortController();
   private currentTurnHandle: VoiceTurnHandle | null = null;
@@ -139,12 +140,12 @@ export class CallController {
    * without blocking the caller.
    */
   private guardianUnavailableForCall = false;
-  /** Active Fish Audio session — tracked so interrupt handling can close it. */
-  private activeFishAbort: AbortController | null = null;
+  /** Active synthesized-TTS session — tracked so interrupt handling can close it. */
+  private activeSynthesisAbort: AbortController | null = null;
 
   constructor(
     callSessionId: string,
-    relay: RelayConnection,
+    transport: CallTransport,
     task: string | null,
     opts?: {
       broadcast?: (msg: ServerMessage) => void;
@@ -153,7 +154,7 @@ export class CallController {
     },
   ) {
     this.callSessionId = callSessionId;
-    this.relay = relay;
+    this.transport = transport;
     this.task = task;
     this.isInbound = !task;
     this.broadcast = opts?.broadcast;
@@ -344,22 +345,58 @@ export class CallController {
     await this.runTurn(`[USER_INSTRUCTION: ${instructionText}]`);
   }
 
+  /**
+   * Handle a barge-in attempt from inbound caller audio.
+   *
+   * Only interrupts the in-flight turn when the assistant is actively
+   * speaking. When the controller is idle or still processing (no TTS
+   * output yet), the barge-in is ignored — this prevents false
+   * interruption on initial inbound media frames that arrive before
+   * the assistant has had a chance to produce its first response.
+   *
+   * @returns `true` if the barge-in was accepted (assistant was speaking),
+   *   `false` if it was ignored (assistant idle or processing).
+   */
+  handleBargeIn(): boolean {
+    if (this.state !== "speaking") {
+      log.debug(
+        {
+          callSessionId: this.callSessionId,
+          state: this.state,
+        },
+        "Barge-in ignored — assistant not speaking",
+      );
+      return false;
+    }
+
+    log.info(
+      { callSessionId: this.callSessionId },
+      "Barge-in accepted — interrupting assistant speech",
+    );
+    this.handleInterrupt();
+    return true;
+  }
+
   /**
    * Handle caller interrupting the assistant's speech.
+   *
+   * This is the hard interrupt path used for explicit teardown and
+   * internal abort scenarios. For barge-in from inbound audio, prefer
+   * {@link handleBargeIn} which gates on the speaking state.
    */
   handleInterrupt(): void {
     const wasSpeaking = this.state === "speaking";
     this.abortCurrentTurn();
     this.llmRunVersion++;
-    // Cancel in-flight Fish Audio synthesis on barge-in
-    if (this.activeFishAbort) {
-      this.activeFishAbort.abort();
-      this.activeFishAbort = null;
+    // Cancel in-flight synthesized TTS on barge-in
+    if (this.activeSynthesisAbort) {
+      this.activeSynthesisAbort.abort();
+      this.activeSynthesisAbort = null;
     }
     // Explicitly terminate the in-progress TTS turn so the relay can
     // immediately hand control back to the caller after barge-in.
     if (wasSpeaking) {
-      this.relay.sendTextToken("", true);
+      this.transport.sendTextToken("", true);
     }
     this.state = "idle";
     // Restart silence detection so a barge-in that never yields a
@@ -386,9 +423,9 @@ export class CallController {
     this.pendingInstructions = [];
     this.llmRunVersion++;
     this.abortCurrentTurn();
-    if (this.activeFishAbort) {
-      this.activeFishAbort.abort();
-      this.activeFishAbort = null;
+    if (this.activeSynthesisAbort) {
+      this.activeSynthesisAbort.abort();
+      this.activeSynthesisAbort = null;
     }
     this.currentTurnPromise = null;
     unregisterCallController(this.callSessionId);
@@ -516,7 +553,7 @@ export class CallController {
         return;
       }
       log.error({ err, callSessionId: this.callSessionId }, "Voice turn error");
-      this.relay.sendTextToken(
+      this.transport.sendTextToken(
         "I'm sorry, I encountered a technical issue. Could you repeat that?",
         true,
       );
@@ -536,11 +573,19 @@ export class CallController {
     runVersion: number,
     runSignal: AbortSignal,
   ): Promise<string> {
-    // Fish Audio TTS routing: when configured, buffer text by sentence
-    // boundaries and synthesize via Fish Audio instead of streaming text
-    // tokens for ElevenLabs TTS.
-    const config = loadConfig();
-    const useFishAudio = isFishAudioTts(config);
+    // Resolve the active TTS provider through the global abstraction.
+    // The catalog's callMode determines the call path: synthesized-play
+    // providers buffer text, synthesize via provider API, and stream
+    // audio chunks to Twilio via play-URL. Native-twilio providers
+    // stream text tokens to the relay for Twilio's built-in TTS.
+    //
+    // When the transport requires WAV (media-stream), request WAV so
+    // the audio store entry and any downstream fetch/transcode receives
+    // PCM that audioBufferToFrames can convert to mu-law.
+    const { provider, useSynthesizedPath, audioFormat } =
+      resolveCallTtsProvider({
+        preferWav: this.transport.requiresWavAudio,
+      });
 
     // Buffer incoming tokens so we can strip control markers ([ASK_GUARDIAN:...], [END_CALL])
     // before they reach TTS. We hold text whenever an unmatched '[' appears, since it
@@ -548,18 +593,18 @@ export class CallController {
     let ttsBuffer = "";
     let fullResponseText = "";
 
-    // When using Fish Audio, we accumulate all text and synthesize
+    // When using the synthesized path, we accumulate all text and synthesize
     // the complete response at the end of the turn (better prosody).
-    let fishAudioTextBuffer = "";
+    let synthesizedTextBuffer = "";
 
     /** Emit a chunk of safe text to the appropriate TTS backend. */
     const emitSafeChunk = (safeText: string): void => {
       const cleaned = sanitizeForTts(safeText);
       if (cleaned.length === 0) return;
-      if (useFishAudio) {
-        fishAudioTextBuffer += cleaned;
+      if (useSynthesizedPath) {
+        synthesizedTextBuffer += cleaned;
       } else {
-        this.relay.sendTextToken(cleaned, false);
+        this.transport.sendTextToken(cleaned, false);
       }
     };
 
@@ -670,48 +715,29 @@ export class CallController {
       emitSafeChunk(ttsBuffer);
     }
 
-    // When using Fish Audio, synthesize the complete response text in a
-    // single REST API call. The full text gives Fish Audio better context
-    // for prosody and intonation. Audio streams back via chunked transfer
-    // encoding and is forwarded to Twilio as it arrives.
-    const sanitizedFishText = sanitizeForTts(fishAudioTextBuffer.trim());
-    if (useFishAudio && sanitizedFishText.length > 0) {
+    // Synthesized-play path: when the active provider supports streaming,
+    // synthesize the complete response text via the provider's streaming
+    // API. The full text gives the provider better context for prosody
+    // and intonation. Audio streams back via chunked transfer encoding
+    // and is forwarded to Twilio as it arrives.
+    const sanitizedSynthText = sanitizeForTts(synthesizedTextBuffer.trim());
+    if (useSynthesizedPath && provider && sanitizedSynthText.length > 0) {
       if (!this.isCurrentRun(runVersion)) return fullResponseText;
-      let handle: ReturnType<typeof createStreamingEntry> | null = null;
-      try {
-        const format = config.fishAudio.format ?? "mp3";
-        handle = createStreamingEntry(format as "mp3" | "wav" | "opus");
-        const baseUrl = getPublicBaseUrl(config);
-        const url = `${baseUrl}/v1/audio/${handle.audioId}`;
-        this.relay.sendPlayUrl(url);
-        const abortController = new AbortController();
-        this.activeFishAbort = abortController;
-        await synthesizeWithFishAudio(
-          sanitizedFishText,
-          config.fishAudio,
-          {
-            onChunk: (chunk) => handle!.push(chunk),
-            signal: abortController.signal,
-          },
-        );
-      } catch (err) {
-        if (err instanceof DOMException && err.name === "AbortError") {
-          log.debug("Fish Audio synthesis aborted (barge-in)");
-        } else {
-          log.error({ err }, "Fish Audio synthesis failed — skipping");
-        }
-      } finally {
-        this.activeFishAbort = null;
-        handle?.finalize();
-      }
+      await this.synthesizeAndStreamAudio(
+        provider,
+        sanitizedSynthText,
+        runVersion,
+        audioFormat,
+      );
     }
 
     // Signal end of this turn's speech.  An empty token with `last: true`
     // tells ConversationRelay to start listening — it does NOT trigger TTS
-    // synthesis.  This is required even when Fish Audio handled all audio
-    // playback, because ConversationRelay still needs the end-of-turn signal
-    // to transition from "assistant speaking" to "caller speaking" state.
-    this.relay.sendTextToken("", true);
+    // synthesis.  This is required even when a synthesized provider handled
+    // all audio playback, because ConversationRelay still needs the
+    // end-of-turn signal to transition from "assistant speaking" to
+    // "caller speaking" state.
+    this.transport.sendTextToken("", true);
 
     // Mark the greeting's first response as awaiting ack
     if (this.lastSentWasOpener && fullResponseText.length > 0) {
@@ -722,6 +748,133 @@ export class CallController {
     return fullResponseText;
   }
 
+  /**
+   * Synthesize text via a streaming TTS provider and forward audio chunks
+   * to Twilio through the audio store / play-URL mechanism.
+   */
+  private async synthesizeAndStreamAudio(
+    provider: TtsProvider,
+    text: string,
+    _runVersion: number,
+    format: "mp3" | "wav" | "opus" = "mp3",
+  ): Promise<void> {
+    let handle: ReturnType<typeof createStreamingEntry> | null = null;
+    let playUrlSent = false;
+    try {
+      // When format is WAV (media-stream transport), request raw PCM from
+      // the provider so the audio bytes match the store's content-type.
+      // Without this, providers like Fish Audio still return mp3 and the
+      // downstream mu-law transcoder fails on the format mismatch.
+      const outputFormat = format === "wav" ? ("pcm" as const) : undefined;
+
+      // Use "pcm" as the store format when requesting PCM output so the
+      // audio store entry's content-type (audio/pcm) matches the raw PCM
+      // bytes providers return. Without this, the store says "audio/wav"
+      // but the bytes have no RIFF header, causing audioBufferToFrames to
+      // fall through to the wrong decode path.
+      const storeFormat = outputFormat ? "pcm" : format;
+      handle = createStreamingEntry(storeFormat);
+      const config = loadConfig();
+      const baseUrl = getPublicBaseUrl(config);
+      const url = `${baseUrl}/v1/audio/${handle.audioId}`;
+      const sendPlayUrlOnce = (): void => {
+        if (playUrlSent) return;
+        this.transport.sendPlayUrl(url);
+        playUrlSent = true;
+      };
+
+      const abortController = new AbortController();
+      this.activeSynthesisAbort = abortController;
+
+      if (provider.synthesizeStream) {
+        let streamedChunk = false;
+        await provider.synthesizeStream(
+          {
+            text,
+            useCase: "phone-call",
+            outputFormat,
+            signal: abortController.signal,
+          },
+          (chunk) => {
+            if (chunk.byteLength === 0) return;
+            if (!streamedChunk) {
+              sendPlayUrlOnce();
+              streamedChunk = true;
+            }
+            handle!.push(chunk);
+          },
+        );
+
+        // Some provider adapters may return a buffer without invoking
+        // onChunk. If that happens, do not leave a dangling unspeakable
+        // turn; degrade to native token TTS below by treating it as no-audio.
+        if (!streamedChunk) {
+          throw new Error("Streaming TTS returned no audio chunks");
+        }
+      } else {
+        // Fallback: buffer-oriented synthesis for providers that don't
+        // implement streaming (shouldn't normally reach here since
+        // useSynthesizedPath is gated on catalog callMode).
+        const result = await provider.synthesize({
+          text,
+          useCase: "phone-call",
+          outputFormat,
+          signal: abortController.signal,
+        });
+        if (result.audio.byteLength === 0) {
+          throw new Error("Buffer TTS returned an empty audio payload");
+        }
+        sendPlayUrlOnce();
+        handle.push(result.audio);
+      }
+    } catch (err) {
+      if (err instanceof DOMException && err.name === "AbortError") {
+        log.debug(
+          { provider: provider.id },
+          "TTS synthesis aborted (barge-in)",
+        );
+      } else {
+        // Extract error class and code for diagnosable log entries.
+        const errName = err instanceof Error ? err.name : String(err);
+        const errCode =
+          err instanceof Error && "code" in err
+            ? (err as Error & { code?: string }).code
+            : undefined;
+
+        // `allowNativeFallback` controls whether the LLM's original
+        // response text should be sent via native Twilio token-based
+        // TTS when synthesis fails. When false (e.g. Deepgram), the
+        // error is re-thrown so the outer catch handler sends a
+        // generic recovery message via native TTS instead — the
+        // caller still hears *something*, but not the LLM's text
+        // rendered in a mismatched voice.
+        const catalogEntry = getCatalogProvider(provider.id as TtsProviderId);
+        if (!catalogEntry.allowNativeFallback) {
+          log.error(
+            { err, provider: provider.id, errName, errCode },
+            "TTS synthesis failed — native fallback disabled for this provider",
+          );
+          throw err;
+        }
+
+        log.error(
+          { err, provider: provider.id, errName, errCode },
+          "TTS synthesis failed — falling back to native token TTS",
+        );
+        // If synthesis fails before any audio has started, degrade to
+        // token-based speech on ConversationRelay so the caller still
+        // hears a response instead of silence. This fallback is only
+        // used for providers whose catalog entry allows native fallback.
+        if (!playUrlSent && !this.transport.requiresWavAudio) {
+          this.transport.sendTextToken(text, false);
+        }
+      }
+    } finally {
+      this.activeSynthesisAbort = null;
+      handle?.finalize();
+    }
+  }
+
   /**
    * Handle post-turn marker detection and dispatch: guardian consultation
    * (ASK_GUARDIAN_APPROVAL / ASK_GUARDIAN), call finalization (END_CALL),
@@ -734,7 +887,9 @@ export class CallController {
     recordCallEvent(this.callSessionId, "assistant_spoke", {
       text: responseText,
     });
-    const spokenText = sanitizeForTts(stripInternalSpeechMarkers(responseText)).trim();
+    const spokenText = sanitizeForTts(
+      stripInternalSpeechMarkers(responseText),
+    ).trim();
     if (spokenText.length > 0) {
       const session = getCallSession(this.callSessionId);
       if (session) {
@@ -952,7 +1107,7 @@ export class CallController {
           currentSession.status !== "cancelled"
         : false;
 
-      this.relay.endSession("Call completed");
+      this.transport.endSession("Call completed");
       updateCallSession(this.callSessionId, {
         status: "completed",
         endedAt: Date.now(),
@@ -1200,7 +1355,7 @@ export class CallController {
           { callSessionId: this.callSessionId },
           "Call duration warning",
         );
-        this.relay.sendTextToken(
+        this.transport.sendTextToken(
           "Just to let you know, we're running low on time for this call.",
           true,
         );
@@ -1212,7 +1367,7 @@ export class CallController {
         { callSessionId: this.callSessionId },
         "Call duration limit reached",
       );
-      this.relay.sendTextToken(
+      this.transport.sendTextToken(
         "I'm sorry, but we've reached the maximum time for this call. Thank you for your time. Goodbye!",
         true,
       );
@@ -1225,7 +1380,7 @@ export class CallController {
             currentSession.status !== "cancelled"
           : false;
 
-        this.relay.endSession("Maximum call duration reached");
+        this.transport.endSession("Maximum call duration reached");
         updateCallSession(this.callSessionId, {
           status: "completed",
           endedAt: Date.now(),
@@ -1274,7 +1429,7 @@ export class CallController {
       // inbound access-request wait (relay state).
       if (
         this.pendingGuardianInput ||
-        this.relay.getConnectionState() === "awaiting_guardian_decision"
+        this.transport.getConnectionState() === "awaiting_guardian_decision"
       ) {
         log.debug(
           { callSessionId: this.callSessionId },
@@ -1286,7 +1441,7 @@ export class CallController {
         { callSessionId: this.callSessionId },
         "Silence timeout triggered",
       );
-      this.relay.sendTextToken("Are you still there?", true);
+      this.transport.sendTextToken("Are you still there?", true);
     }, getSilenceTimeoutMs());
   }
 }

package/src/calls/call-domain.ts

@@ -38,6 +38,7 @@ import {
   getPendingQuestion,
   updateCallSession,
 } from "./call-store.js";
+import { activeMediaStreamSessions } from "./media-stream-server.js";
 import { activeRelayConnections } from "./relay-server.js";
 import { getTwilioConfig } from "./twilio-config.js";
 import { TwilioConversationRelayProvider } from "./twilio-provider.js";
@@ -679,6 +680,14 @@ export async function cancelCall(
     activeRelayConnections.delete(callSessionId);
   }
 
+  // End the media-stream session if active
+  const mediaStreamSession = activeMediaStreamSessions.get(callSessionId);
+  if (mediaStreamSession) {
+    mediaStreamSession.getOutput().endSession(reason);
+    mediaStreamSession.destroy();
+    activeMediaStreamSessions.delete(callSessionId);
+  }
+
   // Clean up controller
   const controller = getCallController(callSessionId);
   if (controller) {