@vellumai/assistant 0.6.3 → 0.6.4

package/src/calls/call-speech-output.ts

@@ -0,0 +1,190 @@
+/**
+ * Provider-aware speech output for deterministic call prompts.
+ *
+ * Deterministic call prompts (verification codes, guardian wait updates,
+ * timeout copy, failure copy, etc.) are routed through the same provider
+ * abstraction used by the call controller so that configured synthesized
+ * providers (e.g. Fish Audio) are respected for all spoken output.
+ *
+ * Two output paths (determined by the catalog's `callMode`):
+ * - **Native**: `callMode: "native-twilio"` — text is sent via
+ *   `sendTextToken()` for Twilio's built-in TTS engine.
+ * - **Synthesized**: `callMode: "synthesized-play"` — text is synthesized
+ *   via the provider API, streamed through the audio store, and played
+ *   via `sendPlayUrl()`.
+ */
+
+import { loadConfig } from "../config/loader.js";
+import { getPublicBaseUrl } from "../inbound/public-ingress-urls.js";
+import { getCatalogProvider } from "../tts/provider-catalog.js";
+import type { TtsProvider, TtsProviderId } from "../tts/types.js";
+import { getLogger } from "../util/logger.js";
+import { createStreamingEntry } from "./audio-store.js";
+import type { CallTransport } from "./call-transport.js";
+import { resolveCallTtsProvider } from "./resolve-call-tts-provider.js";
+
+const log = getLogger("call-speech-output");
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Speak a deterministic text prompt through the active TTS provider.
+ *
+ * For native providers this is equivalent to `relay.sendTextToken(text, true)`
+ * and resolves immediately (synchronous send).
+ *
+ * For synthesized providers this synthesizes audio via the provider API,
+ * sends the play URL to the relay, and resolves once synthesis is complete.
+ * Callers in disconnect/teardown flows should `await` the returned promise
+ * before starting teardown timers so that the play URL is delivered to
+ * Twilio before the session ends. Interactive mid-call callers can
+ * fire-and-forget with `void speakSystemPrompt(...)`.
+ */
+export function speakSystemPrompt(
+  relay: CallTransport,
+  text: string,
+): Promise<void> {
+  // When the transport requires WAV (media-stream), request WAV so
+  // the audio store entry contains PCM that audioBufferToFrames can
+  // transcode to mu-law. Without this, compressed formats (mp3, opus)
+  // are fetched by processFetchUrlItem and produce garbled audio.
+  const { provider, useSynthesizedPath, audioFormat } = resolveCallTtsProvider({
+    preferWav: relay.requiresWavAudio,
+  });
+
+  if (!useSynthesizedPath || !provider) {
+    // Native path — send text for Twilio's built-in TTS.
+    relay.sendTextToken(text, true);
+    return Promise.resolve();
+  }
+
+  // Synthesized path — synthesize audio and send play URL.
+  return synthesizeAndPlay(relay, provider, text, audioFormat);
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Synthesize text via a streaming TTS provider and send the play URL
+ * to the relay.
+ *
+ * On synthesis failure the behavior depends on the provider:
+ * - Providers with a native Twilio TTS fallback (e.g. Fish Audio) fall
+ *   back to `sendTextToken(text)` so the caller still hears the message.
+ * - Providers without a native fallback (e.g. Deepgram) log the error
+ *   and send only an empty end-of-turn signal — the caller hears nothing
+ *   but the relay transitions back to listening state.
+ */
+async function synthesizeAndPlay(
+  relay: CallTransport,
+  provider: TtsProvider,
+  text: string,
+  format: "mp3" | "wav" | "opus",
+): Promise<void> {
+  let handle: ReturnType<typeof createStreamingEntry> | null = null;
+  let playUrlSent = false;
+  try {
+    // When format is WAV (media-stream transport), request raw PCM from
+    // the provider so the audio bytes match the store's content-type.
+    // Without this, providers like Fish Audio still return mp3 and the
+    // downstream mu-law transcoder fails on the format mismatch.
+    const outputFormat = format === "wav" ? ("pcm" as const) : undefined;
+
+    // Use "pcm" as the store format when requesting PCM output so the
+    // audio store entry's content-type (audio/pcm) matches the raw PCM
+    // bytes providers return. Without this, the store says "audio/wav"
+    // but the bytes have no RIFF header, causing audioBufferToFrames to
+    // fall through to the wrong decode path.
+    const storeFormat = outputFormat ? "pcm" : format;
+    handle = createStreamingEntry(storeFormat);
+    const config = loadConfig();
+    const baseUrl = getPublicBaseUrl(config);
+    const url = `${baseUrl}/v1/audio/${handle.audioId}`;
+    const sendPlayUrlOnce = (): void => {
+      if (playUrlSent) return;
+      relay.sendPlayUrl(url);
+      playUrlSent = true;
+    };
+
+    if (provider.synthesizeStream) {
+      let streamedChunk = false;
+      await provider.synthesizeStream(
+        { text, useCase: "phone-call", outputFormat },
+        (chunk) => {
+          if (chunk.byteLength === 0) return;
+          if (!streamedChunk) {
+            sendPlayUrlOnce();
+            streamedChunk = true;
+          }
+          handle!.push(chunk);
+        },
+      );
+      if (!streamedChunk) {
+        throw new Error("Streaming TTS returned no audio chunks");
+      }
+    } else {
+      const result = await provider.synthesize({
+        text,
+        useCase: "phone-call",
+        outputFormat,
+      });
+      if (result.audio.byteLength === 0) {
+        throw new Error("Buffer TTS returned an empty audio payload");
+      }
+      sendPlayUrlOnce();
+      handle.push(result.audio);
+    }
+
+    // Signal end of this turn's speech.  An empty token with `last: true`
+    // tells ConversationRelay to start listening — it does NOT trigger TTS
+    // synthesis.  This is required even when a synthesized provider handled
+    // all audio playback, because ConversationRelay still needs the
+    // end-of-turn signal to transition from "assistant speaking" to
+    // "caller speaking" state.
+    relay.sendTextToken("", true);
+  } catch (err) {
+    // Extract error class and code for diagnosable log entries.
+    const errName = err instanceof Error ? err.name : String(err);
+    const errCode =
+      err instanceof Error && "code" in err
+        ? (err as Error & { code?: string }).code
+        : undefined;
+
+    // `allowNativeFallback` controls whether the system prompt text
+    // should be sent via native Twilio token-based TTS when synthesis
+    // fails. When false (e.g. Deepgram), the design choice is to
+    // send only an end-of-turn signal — the caller hears nothing for
+    // this prompt — rather than degrading to a mismatched voice.
+    // Callers use fire-and-forget (`void speakSystemPrompt(...)`) so
+    // throwing here would produce an unhandled promise rejection.
+    const catalogEntry = getCatalogProvider(provider.id as TtsProviderId);
+    if (!catalogEntry.allowNativeFallback) {
+      log.error(
+        { err, provider: provider.id, errName, errCode },
+        "System prompt TTS synthesis failed — native fallback disabled for this provider",
+      );
+      // Send the end-of-turn signal so ConversationRelay transitions from
+      // "assistant speaking" to "caller speaking" state. Without this, the
+      // relay hangs waiting for the prompt to complete and the caller
+      // cannot interact.
+      relay.sendTextToken("", true);
+      return;
+    }
+
+    log.error(
+      { err, provider: provider.id, errName, errCode },
+      "System prompt TTS synthesis failed — falling back to native TTS",
+    );
+    // Fallback: send text via native TTS so the caller still hears the message.
+    // sendTextToken with last:true includes the end-of-turn signal inherently.
+    // This fallback is only used for providers whose catalog entry allows
+    // native fallback.
+    relay.sendTextToken(text, true);
+  } finally {
+    handle?.finalize();
+  }
+}

package/src/calls/call-transport.ts

@@ -0,0 +1,77 @@
+/**
+ * Transport interface consumed by CallController for sending voice output
+ * and controlling call lifecycle.
+ *
+ * Decouples the controller from any specific wire protocol (e.g.
+ * ConversationRelay) so that alternative transports (media-stream, etc.)
+ * can be introduced without modifying controller logic.
+ */
+
+// ── Transport interface ──────────────────────────────────────────────
+
+/**
+ * Minimal output surface that CallController uses to send speech,
+ * audio, and lifecycle signals to the caller.
+ */
+export interface CallTransport {
+  /**
+   * Send a text token for TTS playback. When `last` is true the
+   * transport should signal end-of-turn to the caller.
+   */
+  sendTextToken(token: string, last: boolean): void;
+
+  /**
+   * Send a pre-synthesized audio URL for playback.
+   */
+  sendPlayUrl(url: string): void;
+
+  /**
+   * Signal the transport to end the call session.
+   */
+  endSession(reason?: string): void;
+
+  /**
+   * Return the current connection-level state. The controller uses this
+   * to suppress silence nudges during guardian wait states.
+   */
+  getConnectionState(): string;
+
+  /**
+   * When true, the transport requires WAV (PCM) audio for playback.
+   *
+   * The media-stream transport sets this because its mu-law transcoder
+   * can only decode WAV (raw PCM) — compressed formats (mp3, opus)
+   * produce garbled audio. The call controller uses this to request
+   * WAV from TTS providers and the audio store.
+   */
+  readonly requiresWavAudio?: boolean;
+}
+
+// ── ConversationRelay adapter ────────────────────────────────────────
+
+import type { RelayConnection } from "./relay-server.js";
+
+/**
+ * Adapts a RelayConnection (Twilio ConversationRelay WebSocket) to the
+ * CallTransport interface. All calls are forwarded 1:1 — no behavioral
+ * changes from the pre-abstraction path.
+ */
+export class ConversationRelayTransport implements CallTransport {
+  constructor(private relay: RelayConnection) {}
+
+  sendTextToken(token: string, last: boolean): void {
+    this.relay.sendTextToken(token, last);
+  }
+
+  sendPlayUrl(url: string): void {
+    this.relay.sendPlayUrl(url);
+  }
+
+  endSession(reason?: string): void {
+    this.relay.endSession(reason);
+  }
+
+  getConnectionState(): string {
+    return this.relay.getConnectionState();
+  }
+}

package/src/calls/media-stream-audio-transcode.ts

@@ -0,0 +1,173 @@
+/**
+ * Audio transcoding helpers for media-stream outbound playback.
+ *
+ * Twilio media streams send and receive audio as base64-encoded mu-law
+ * (audio/x-mulaw) at 8 kHz mono. This module provides utilities for:
+ *
+ * 1. Converting linear PCM audio (from TTS providers) to mu-law encoding.
+ * 2. Chunking a contiguous audio buffer into Twilio-compatible frame sizes.
+ * 3. Encoding frames as base64 strings ready for the `media` outbound command.
+ *
+ * The chunk size is aligned to Twilio's expected frame duration (~20 ms at
+ * 8 kHz = 160 samples per frame). Larger payloads are split into multiple
+ * frames to avoid buffering delays on the Twilio side.
+ *
+ * @module
+ */
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/**
+ * Twilio media streams expect 20 ms frames at 8 kHz mono mu-law.
+ * 8000 samples/sec * 0.020 sec = 160 samples per frame.
+ * Each mu-law sample is 1 byte, so each frame is 160 bytes.
+ */
+export const MULAW_FRAME_SIZE = 160;
+
+/**
+ * Bias constant used in the linear-to-mu-law compression formula
+ * (ITU-T G.711).
+ */
+const MULAW_BIAS = 0x84;
+
+/** Maximum value for the mu-law compression input. */
+const MULAW_CLIP = 32635;
+
+/**
+ * Lookup table for the segment number in mu-law encoding.
+ * Maps magnitude ranges to segment indices.
+ */
+const MULAW_SEG_TABLE = [0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3];
+
+// ---------------------------------------------------------------------------
+// Linear PCM to mu-law conversion
+// ---------------------------------------------------------------------------
+
+/**
+ * Compress a single 16-bit linear PCM sample to 8-bit mu-law.
+ *
+ * Implements the ITU-T G.711 mu-law encoding algorithm. The input is a
+ * signed 16-bit integer (range: -32768 to +32767). The output is an
+ * unsigned 8-bit mu-law value.
+ */
+export function linearToMulaw(sample: number): number {
+  // Determine sign and clamp magnitude
+  const sign = (sample >> 8) & 0x80;
+  if (sign !== 0) sample = -sample;
+  if (sample > MULAW_CLIP) sample = MULAW_CLIP;
+  sample += MULAW_BIAS;
+
+  // Determine segment
+  const exponent = segmentSearch(sample);
+  const mantissa = (sample >> (exponent + 3)) & 0x0f;
+  const mulawByte = ~(sign | (exponent << 4) | mantissa) & 0xff;
+
+  return mulawByte;
+}
+
+function segmentSearch(val: number): number {
+  let shifted = val >> 7;
+  if (shifted >= 16) {
+    shifted >>= 4;
+    if (shifted >= 16) return 7;
+    return MULAW_SEG_TABLE[shifted] + 4;
+  }
+  return MULAW_SEG_TABLE[shifted];
+}
+
+// ---------------------------------------------------------------------------
+// PCM buffer to mu-law buffer conversion
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert a buffer of 16-bit signed little-endian PCM samples to mu-law.
+ *
+ * @param pcm - Raw PCM audio buffer. Every 2 bytes is one sample (LE).
+ * @returns A Buffer of mu-law encoded bytes (half the length of the input).
+ */
+export function pcm16ToMulaw(pcm: Uint8Array): Buffer {
+  const sampleCount = Math.floor(pcm.length / 2);
+  const mulaw = Buffer.alloc(sampleCount);
+  const view = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+
+  for (let i = 0; i < sampleCount; i++) {
+    const sample = view.getInt16(i * 2, true); // little-endian
+    mulaw[i] = linearToMulaw(sample);
+  }
+
+  return mulaw;
+}
+
+// ---------------------------------------------------------------------------
+// Chunking
+// ---------------------------------------------------------------------------
+
+/**
+ * Split a mu-law audio buffer into base64-encoded frames suitable for
+ * Twilio media stream `media` commands.
+ *
+ * Each frame is {@link MULAW_FRAME_SIZE} bytes (160 bytes = 20 ms at 8 kHz).
+ * The final frame may be shorter if the buffer length is not an exact
+ * multiple of the frame size.
+ *
+ * @param mulawBuffer - Contiguous mu-law audio bytes.
+ * @returns Array of base64-encoded frame strings.
+ */
+export function chunkMulawToBase64Frames(mulawBuffer: Buffer): string[] {
+  const frames: string[] = [];
+  let offset = 0;
+
+  while (offset < mulawBuffer.length) {
+    const end = Math.min(offset + MULAW_FRAME_SIZE, mulawBuffer.length);
+    const frame = mulawBuffer.subarray(offset, end);
+    frames.push(Buffer.from(frame).toString("base64"));
+    offset = end;
+  }
+
+  return frames;
+}
+
+// ---------------------------------------------------------------------------
+// High-level: raw audio bytes to sendable base64 frames
+// ---------------------------------------------------------------------------
+
+/**
+ * Accepted audio formats for transcoding into mu-law frames.
+ *
+ * Currently only `mulaw` (already in the target format) and `pcm16` (16-bit
+ * signed LE PCM at 8 kHz) are supported. Higher-level formats (mp3, wav,
+ * opus) require decoding to PCM first -- callers should handle that
+ * conversion before invoking this helper.
+ */
+export type TranscodeInputFormat = "mulaw" | "pcm16";
+
+/**
+ * Transcode raw audio bytes into Twilio-compatible base64-encoded mu-law
+ * frames.
+ *
+ * @param audio - Raw audio bytes in the specified format.
+ * @param format - The format of the input audio.
+ * @returns Array of base64-encoded mu-law frames ready for
+ *   `MediaStreamOutput.sendAudioPayload()`.
+ */
+export function transcodeToMulawFrames(
+  audio: Uint8Array,
+  format: TranscodeInputFormat,
+): string[] {
+  let mulawBuffer: Buffer;
+
+  switch (format) {
+    case "mulaw":
+      mulawBuffer = Buffer.from(audio);
+      break;
+    case "pcm16":
+      mulawBuffer = pcm16ToMulaw(audio);
+      break;
+    default:
+      throw new Error(`Unsupported transcode format: ${format as string}`);
+  }
+
+  return chunkMulawToBase64Frames(mulawBuffer);
+}