@vellumai/assistant 0.6.3 → 0.6.4

package/src/tts/provider-registry.ts

@@ -0,0 +1,73 @@
+/**
+ * Runtime TTS provider registry.
+ *
+ * Providers self-register at startup via `registerTtsProvider`. Callers
+ * resolve a provider by ID with `getTtsProvider`, which throws an explicit
+ * error for unknown IDs so misconfiguration surfaces immediately rather
+ * than producing silent fallback behavior.
+ */
+
+import type { TtsProvider, TtsProviderId } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Internal state
+// ---------------------------------------------------------------------------
+
+/** Insertion-ordered map of registered providers. */
+const providers = new Map<TtsProviderId, TtsProvider>();
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Register a TTS provider.
+ *
+ * @throws if a provider with the same ID is already registered — this
+ *   prevents silent overwrites caused by duplicate registrations.
+ */
+export function registerTtsProvider(provider: TtsProvider): void {
+  if (providers.has(provider.id)) {
+    throw new Error(
+      `TTS provider "${provider.id}" is already registered. ` +
+        "Duplicate registrations are not allowed.",
+    );
+  }
+  providers.set(provider.id, provider);
+}
+
+/**
+ * Look up a registered TTS provider by ID.
+ *
+ * @throws if no provider with the given ID has been registered.
+ */
+export function getTtsProvider(id: TtsProviderId): TtsProvider {
+  const provider = providers.get(id);
+  if (!provider) {
+    const known = [...providers.keys()];
+    const knownList =
+      known.length > 0 ? ` Known providers: ${known.join(", ")}` : "";
+    throw new Error(`Unknown TTS provider "${id}".${knownList}`);
+  }
+  return provider;
+}
+
+/**
+ * List all registered providers in deterministic (registration) order.
+ */
+export function listTtsProviders(): TtsProvider[] {
+  return [...providers.values()];
+}
+
+// ---------------------------------------------------------------------------
+// Test helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Clear all registered providers.
+ *
+ * **Test-only** — must not be called in production code.
+ */
+export function _resetTtsProviderRegistry(): void {
+  providers.clear();
+}

package/src/tts/providers/deepgram-provider.ts

@@ -0,0 +1,219 @@
+/**
+ * Deepgram TTS provider adapter.
+ *
+ * Wraps the Deepgram REST text-to-speech API (`/v1/speak`) behind the uniform
+ * {@link TtsProvider} interface. Reads the API key from the secure credential
+ * store using the shared `deepgram` bare key (shared with STT) and the model
+ * configuration from `services.tts.providers.deepgram` config section.
+ */
+
+import { getConfig } from "../../config/loader.js";
+import type { TtsDeepgramProviderConfig } from "../../config/schemas/tts.js";
+import { getProviderKeyAsync } from "../../security/secure-keys.js";
+import { getLogger } from "../../util/logger.js";
+import type {
+  TtsProvider,
+  TtsProviderCapabilities,
+  TtsSynthesisRequest,
+  TtsSynthesisResult,
+} from "../types.js";
+
+const log = getLogger("tts:deepgram");
+
+// ---------------------------------------------------------------------------
+// Error types
+// ---------------------------------------------------------------------------
+
+export type DeepgramTtsErrorCode =
+  | "DEEPGRAM_TTS_NO_API_KEY"
+  | "DEEPGRAM_TTS_HTTP_ERROR"
+  | "DEEPGRAM_TTS_EMPTY_RESPONSE"
+  | "DEEPGRAM_TTS_REQUEST_FAILED";
+
+export class DeepgramTtsError extends Error {
+  readonly code: DeepgramTtsErrorCode;
+  readonly statusCode?: number;
+
+  constructor(
+    code: DeepgramTtsErrorCode,
+    message: string,
+    statusCode?: number,
+  ) {
+    super(message);
+    this.name = "DeepgramTtsError";
+    this.code = code;
+    this.statusCode = statusCode;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const DEEPGRAM_API_BASE = "https://api.deepgram.com";
+
+/** Map from Deepgram encoding names to MIME content types. */
+const FORMAT_CONTENT_TYPE: Record<string, string> = {
+  mp3: "audio/mpeg",
+  wav: "audio/wav",
+  opus: "audio/opus",
+  linear16: "audio/pcm",
+};
+
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+
+/** Parameters for Deepgram's `/v1/speak` encoding query string. */
+interface DeepgramOutputParams {
+  /** Deepgram encoding name (e.g. `mp3`, `linear16`, `opus`). */
+  encoding: string;
+  /** Container override (`wav` or `none`). Omitted lets Deepgram choose. */
+  container?: string;
+  /** Sample rate in Hz. Required for raw PCM to avoid Deepgram's 24 kHz default. */
+  sample_rate?: number;
+  /** Content-type key for the FORMAT_CONTENT_TYPE lookup. */
+  contentTypeKey: string;
+}
+
+/**
+ * Resolve the Deepgram output encoding, container, and sample rate based on
+ * the synthesis request and provider config.
+ *
+ * **PCM path** (`outputFormat: "pcm"`):
+ *   The media-stream transport needs raw headerless PCM for mu-law transcoding.
+ *   We request `encoding=linear16&container=none&sample_rate=16000` — 16-bit
+ *   signed little-endian at 16 kHz with no WAV header. This matches the
+ *   ElevenLabs `pcm_16000` convention and the downstream
+ *   `audioBufferToFrames` expectation (16 kHz -> 8 kHz downsample).
+ *
+ * **WAV path** (`config.format === "wav"`):
+ *   Deepgram treats WAV as a container, not an encoding. We translate to
+ *   `encoding=linear16&container=wav` so the API returns a valid WAV file.
+ *
+ * **Other formats** (mp3, opus):
+ *   Passed through directly as encoding values.
+ */
+function resolveOutputParams(
+  request: TtsSynthesisRequest,
+  config: TtsDeepgramProviderConfig,
+): DeepgramOutputParams {
+  if (request.outputFormat === "pcm") {
+    return {
+      encoding: "linear16",
+      container: "none",
+      sample_rate: 16_000,
+      contentTypeKey: "linear16",
+    };
+  }
+
+  if (config.format === "wav") {
+    return {
+      encoding: "linear16",
+      container: "wav",
+      contentTypeKey: "wav",
+    };
+  }
+
+  return { encoding: config.format, contentTypeKey: config.format };
+}
+
+export function createDeepgramProvider(): TtsProvider {
+  const capabilities: TtsProviderCapabilities = {
+    supportsStreaming: false,
+    supportedFormats: ["mp3", "wav", "opus"],
+  };
+
+  return {
+    id: "deepgram",
+    capabilities,
+
+    async synthesize(
+      request: TtsSynthesisRequest,
+    ): Promise<TtsSynthesisResult> {
+      const apiKey = await getProviderKeyAsync("deepgram");
+      if (!apiKey) {
+        throw new DeepgramTtsError(
+          "DEEPGRAM_TTS_NO_API_KEY",
+          "Deepgram API key not configured. " +
+            "Add it in Settings → Voice or via: assistant keys set deepgram <key>",
+        );
+      }
+
+      const config = getConfig().services.tts.providers.deepgram;
+      const outputParams = resolveOutputParams(request, config);
+      const model = config.model;
+
+      const params = new URLSearchParams({
+        model,
+        encoding: outputParams.encoding,
+      });
+      if (outputParams.container) {
+        params.set("container", outputParams.container);
+      }
+      if (outputParams.sample_rate != null) {
+        params.set("sample_rate", String(outputParams.sample_rate));
+      }
+      const url = `${DEEPGRAM_API_BASE}/v1/speak?${params.toString()}`;
+
+      log.info(
+        {
+          model,
+          encoding: outputParams.encoding,
+          container: outputParams.container,
+          textLength: request.text.length,
+        },
+        "Starting Deepgram TTS synthesis",
+      );
+
+      let response: Response;
+      try {
+        response = await fetch(url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Token ${apiKey}`,
+          },
+          body: JSON.stringify({ text: request.text }),
+          signal: request.signal,
+        });
+      } catch (err) {
+        if (err instanceof Error && err.name === "AbortError") throw err;
+        throw new DeepgramTtsError(
+          "DEEPGRAM_TTS_REQUEST_FAILED",
+          `Deepgram TTS request failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+
+      if (!response.ok) {
+        const errorText = await response.text().catch(() => "");
+        throw new DeepgramTtsError(
+          "DEEPGRAM_TTS_HTTP_ERROR",
+          `Deepgram TTS returned ${response.status}: ${errorText}`,
+          response.status,
+        );
+      }
+
+      const arrayBuffer = await response.arrayBuffer();
+      if (arrayBuffer.byteLength === 0) {
+        throw new DeepgramTtsError(
+          "DEEPGRAM_TTS_EMPTY_RESPONSE",
+          "Deepgram TTS returned an empty audio response",
+        );
+      }
+
+      const contentType =
+        FORMAT_CONTENT_TYPE[outputParams.contentTypeKey] ?? "audio/mpeg";
+
+      log.debug(
+        { bytes: arrayBuffer.byteLength },
+        "Deepgram TTS synthesis complete",
+      );
+
+      return {
+        audio: Buffer.from(arrayBuffer),
+        contentType,
+      };
+    },
+  };
+}

package/src/tts/providers/elevenlabs-provider.ts

@@ -0,0 +1,211 @@
+/**
+ * ElevenLabs TTS provider adapter.
+ *
+ * Wraps the ElevenLabs REST text-to-speech API (`/v1/text-to-speech/:voiceId`)
+ * behind the uniform {@link TtsProvider} interface. Reads the API key from the
+ * secure credential store (`elevenlabs/api_key`) and the voice configuration
+ * from `services.tts.providers.elevenlabs` config section.
+ */
+
+import { getConfig } from "../../config/loader.js";
+import { DEFAULT_ELEVENLABS_VOICE_ID } from "../../config/schemas/elevenlabs.js";
+import type { TtsElevenLabsProviderConfig } from "../../config/schemas/tts.js";
+import { credentialKey } from "../../security/credential-key.js";
+import { getSecureKeyAsync } from "../../security/secure-keys.js";
+import { getLogger } from "../../util/logger.js";
+import type {
+  TtsProvider,
+  TtsProviderCapabilities,
+  TtsSynthesisRequest,
+  TtsSynthesisResult,
+} from "../types.js";
+
+const log = getLogger("tts:elevenlabs");
+
+// ---------------------------------------------------------------------------
+// Error types
+// ---------------------------------------------------------------------------
+
+export type ElevenLabsTtsErrorCode =
+  | "ELEVENLABS_TTS_NO_API_KEY"
+  | "ELEVENLABS_TTS_NO_VOICE_ID"
+  | "ELEVENLABS_TTS_HTTP_ERROR"
+  | "ELEVENLABS_TTS_EMPTY_RESPONSE"
+  | "ELEVENLABS_TTS_REQUEST_FAILED";
+
+export class ElevenLabsTtsError extends Error {
+  readonly code: ElevenLabsTtsErrorCode;
+  readonly statusCode?: number;
+
+  constructor(
+    code: ElevenLabsTtsErrorCode,
+    message: string,
+    statusCode?: number,
+  ) {
+    super(message);
+    this.name = "ElevenLabsTtsError";
+    this.code = code;
+    this.statusCode = statusCode;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const ELEVENLABS_API_BASE = "https://api.elevenlabs.io";
+
+/** Map from request output format identifiers to MIME content types. */
+const FORMAT_CONTENT_TYPE: Record<string, string> = {
+  mp3_44100_128: "audio/mpeg",
+  mp3_22050_32: "audio/mpeg",
+  pcm_16000: "audio/pcm",
+  pcm_22050: "audio/pcm",
+  pcm_24000: "audio/pcm",
+  pcm_44100: "audio/pcm",
+  ulaw_8000: "audio/basic",
+};
+
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the effective voice ID for a synthesis request.
+ *
+ * Priority: request-level `voiceId` > config `voiceId` > built-in default.
+ */
+function resolveVoiceId(
+  request: TtsSynthesisRequest,
+  config: TtsElevenLabsProviderConfig,
+): string {
+  const voiceId =
+    request.voiceId?.trim() || config.voiceId || DEFAULT_ELEVENLABS_VOICE_ID;
+  if (!voiceId) {
+    throw new ElevenLabsTtsError(
+      "ELEVENLABS_TTS_NO_VOICE_ID",
+      "No voice ID provided and no default configured. " +
+        "Set services.tts.providers.elevenlabs.voiceId in config or pass voiceId in the request.",
+    );
+  }
+  return voiceId;
+}
+
+/**
+ * Choose the ElevenLabs output format based on the use case and optional
+ * format hint.
+ *
+ * When the caller requests `outputFormat: "pcm"` (e.g. the media-stream
+ * transport which needs raw PCM for mu-law transcoding), we use `pcm_16000`
+ * — 16-bit signed little-endian at 16 kHz. The media-stream transport's
+ * `audioBufferToFrames` handles the 16 kHz -> 8 kHz downsample.
+ *
+ * Otherwise:
+ * - Phone calls benefit from lower-latency, smaller payloads (mp3 at 22050/32).
+ * - Message playback uses higher quality (mp3 at 44100/128).
+ */
+function resolveOutputFormat(request: TtsSynthesisRequest): string {
+  if (request.outputFormat === "pcm") {
+    return "pcm_16000";
+  }
+  return request.useCase === "phone-call" ? "mp3_22050_32" : "mp3_44100_128";
+}
+
+export function createElevenLabsProvider(): TtsProvider {
+  const capabilities: TtsProviderCapabilities = {
+    supportsStreaming: false,
+    supportedFormats: ["mp3", "pcm"],
+  };
+
+  return {
+    id: "elevenlabs",
+    capabilities,
+
+    async synthesize(
+      request: TtsSynthesisRequest,
+    ): Promise<TtsSynthesisResult> {
+      const apiKey = await getSecureKeyAsync(
+        credentialKey("elevenlabs", "api_key"),
+      );
+      if (!apiKey) {
+        throw new ElevenLabsTtsError(
+          "ELEVENLABS_TTS_NO_API_KEY",
+          "ElevenLabs API key not configured. " +
+            "Add it in Settings → Voice or via: assistant credentials set --service elevenlabs --field api_key <key>",
+        );
+      }
+
+      const config = getConfig().services.tts.providers.elevenlabs;
+      const voiceId = resolveVoiceId(request, config);
+      const outputFormat = resolveOutputFormat(request);
+
+      const url = `${ELEVENLABS_API_BASE}/v1/text-to-speech/${voiceId}`;
+
+      const body: Record<string, unknown> = {
+        text: request.text,
+        model_id: config.voiceModelId?.trim() || "eleven_multilingual_v2",
+        voice_settings: {
+          stability: config.stability,
+          similarity_boost: config.similarityBoost,
+          speed: config.speed,
+        },
+      };
+
+      log.info(
+        { voiceId, outputFormat, textLength: request.text.length },
+        "Starting ElevenLabs TTS synthesis",
+      );
+
+      const acceptType = FORMAT_CONTENT_TYPE[outputFormat] ?? "audio/mpeg";
+
+      let response: Response;
+      try {
+        response = await fetch(`${url}?output_format=${outputFormat}`, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "xi-api-key": apiKey,
+            Accept: acceptType,
+          },
+          body: JSON.stringify(body),
+          signal: request.signal,
+        });
+      } catch (err) {
+        if (err instanceof Error && err.name === "AbortError") throw err;
+        throw new ElevenLabsTtsError(
+          "ELEVENLABS_TTS_REQUEST_FAILED",
+          `ElevenLabs TTS request failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+
+      if (!response.ok) {
+        const errorText = await response.text().catch(() => "");
+        throw new ElevenLabsTtsError(
+          "ELEVENLABS_TTS_HTTP_ERROR",
+          `ElevenLabs TTS returned ${response.status}: ${errorText}`,
+          response.status,
+        );
+      }
+
+      const arrayBuffer = await response.arrayBuffer();
+      if (arrayBuffer.byteLength === 0) {
+        throw new ElevenLabsTtsError(
+          "ELEVENLABS_TTS_EMPTY_RESPONSE",
+          "ElevenLabs TTS returned an empty audio response",
+        );
+      }
+
+      const contentType = FORMAT_CONTENT_TYPE[outputFormat] ?? "audio/mpeg";
+
+      log.debug(
+        { bytes: arrayBuffer.byteLength },
+        "ElevenLabs TTS synthesis complete",
+      );
+
+      return {
+        audio: Buffer.from(arrayBuffer),
+        contentType,
+      };
+    },
+  };
+}

package/src/tts/providers/fish-audio-provider.ts

@@ -0,0 +1,183 @@
+/**
+ * Fish Audio TTS provider adapter.
+ *
+ * Wraps the existing {@link synthesizeWithFishAudio} function behind the
+ * uniform {@link TtsProvider} interface, preserving its streaming chunk
+ * callbacks for real-time call playback.
+ *
+ * Config comes from `services.tts.providers['fish-audio']`. The API key is read
+ * from the secure credential store (`fish-audio/api_key`) by the underlying
+ * client.
+ */
+
+import { synthesizeWithFishAudio } from "../../calls/fish-audio-client.js";
+import { getConfig } from "../../config/loader.js";
+import type { TtsFishAudioProviderConfig } from "../../config/schemas/tts.js";
+import { getLogger } from "../../util/logger.js";
+import type {
+  TtsProvider,
+  TtsProviderCapabilities,
+  TtsSynthesisRequest,
+  TtsSynthesisResult,
+} from "../types.js";
+
+const log = getLogger("tts:fish-audio");
+
+// ---------------------------------------------------------------------------
+// Error types
+// ---------------------------------------------------------------------------
+
+export type FishAudioTtsErrorCode =
+  | "FISH_AUDIO_TTS_NO_REFERENCE_ID"
+  | "FISH_AUDIO_TTS_SYNTHESIS_FAILED";
+
+export class FishAudioTtsError extends Error {
+  readonly code: FishAudioTtsErrorCode;
+
+  constructor(code: FishAudioTtsErrorCode, message: string) {
+    super(message);
+    this.name = "FishAudioTtsError";
+    this.code = code;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Map Fish Audio format names to MIME content types. */
+const FORMAT_CONTENT_TYPE: Record<string, string> = {
+  mp3: "audio/mpeg",
+  wav: "audio/wav",
+  opus: "audio/opus",
+};
+
+/**
+ * Resolve the effective reference ID.
+ *
+ * Priority: request-level `voiceId` > config `referenceId`.
+ */
+function resolveReferenceId(
+  request: TtsSynthesisRequest,
+  config: TtsFishAudioProviderConfig,
+): string {
+  const referenceId = request.voiceId?.trim() || config.referenceId;
+  if (!referenceId) {
+    throw new FishAudioTtsError(
+      "FISH_AUDIO_TTS_NO_REFERENCE_ID",
+      "No Fish Audio reference ID provided. " +
+        "Set services.tts.providers.fish-audio.referenceId in config or pass voiceId in the request.",
+    );
+  }
+  return referenceId;
+}
+
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+
+export function createFishAudioProvider(): TtsProvider {
+  const capabilities: TtsProviderCapabilities = {
+    supportsStreaming: true,
+    supportedFormats: ["mp3", "wav", "opus"],
+  };
+
+  return {
+    id: "fish-audio",
+    capabilities,
+
+    async synthesize(
+      request: TtsSynthesisRequest,
+    ): Promise<TtsSynthesisResult> {
+      const config = getConfig().services.tts.providers["fish-audio"];
+      const referenceId = resolveReferenceId(request, config);
+
+      // When PCM output is requested, override to WAV. Fish Audio
+      // doesn't support raw PCM, but WAV gives us PCM in a container
+      // that audioBufferToFrames can extract.
+      const effectiveFormat =
+        request.outputFormat === "pcm" ? "wav" : config.format;
+
+      // Build an effective config with the resolved reference ID
+      // and the potentially overridden format.
+      const effectiveConfig: TtsFishAudioProviderConfig = {
+        ...config,
+        referenceId,
+        format: effectiveFormat,
+      };
+
+      log.info(
+        {
+          referenceId,
+          format: effectiveFormat,
+          textLength: request.text.length,
+        },
+        "Starting Fish Audio TTS synthesis",
+      );
+
+      let audio: Buffer;
+      try {
+        audio = await synthesizeWithFishAudio(request.text, effectiveConfig, {
+          signal: request.signal,
+        });
+      } catch (err) {
+        if (err instanceof Error && err.name === "AbortError") throw err;
+        throw new FishAudioTtsError(
+          "FISH_AUDIO_TTS_SYNTHESIS_FAILED",
+          `Fish Audio TTS synthesis failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+
+      const contentType = FORMAT_CONTENT_TYPE[effectiveFormat] ?? "audio/mpeg";
+
+      return { audio, contentType };
+    },
+
+    async synthesizeStream(
+      request: TtsSynthesisRequest,
+      onChunk: (chunk: Uint8Array) => void,
+    ): Promise<TtsSynthesisResult> {
+      const config = getConfig().services.tts.providers["fish-audio"];
+      const referenceId = resolveReferenceId(request, config);
+
+      // When PCM output is requested, override to WAV. Fish Audio
+      // doesn't support raw PCM, but WAV gives us PCM in a container
+      // that audioBufferToFrames can extract.
+      const effectiveFormat =
+        request.outputFormat === "pcm" ? "wav" : config.format;
+
+      const effectiveConfig: TtsFishAudioProviderConfig = {
+        ...config,
+        referenceId,
+        format: effectiveFormat,
+      };
+
+      log.info(
+        {
+          referenceId,
+          format: effectiveFormat,
+          textLength: request.text.length,
+        },
+        "Starting Fish Audio TTS streaming synthesis",
+      );
+
+      let audio: Buffer;
+      try {
+        audio = await synthesizeWithFishAudio(request.text, effectiveConfig, {
+          onChunk,
+          signal: request.signal,
+        });
+      } catch (err) {
+        if (err instanceof Error && err.name === "AbortError") throw err;
+        throw new FishAudioTtsError(
+          "FISH_AUDIO_TTS_SYNTHESIS_FAILED",
+          `Fish Audio TTS streaming synthesis failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+
+      const contentType = FORMAT_CONTENT_TYPE[effectiveFormat] ?? "audio/mpeg";
+
+      return { audio, contentType };
+    },
+  };
+}