@vellumai/assistant 0.10.7 → 0.10.8-dev.202607102228.5945895

package/src/tts/providers/fish-audio-provider.ts

@@ -10,10 +10,14 @@
  * client.
  */
 
-import { synthesizeWithFishAudio } from "../../calls/fish-audio-client.js";
+import {
+  type FishAudioSynthesisConfig,
+  synthesizeWithFishAudio,
+} from "../../calls/fish-audio-client.js";
 import { getConfig } from "../../config/loader.js";
 import type { TtsFishAudioProviderConfig } from "../../config/schemas/tts.js";
 import { getLogger } from "../../util/logger.js";
+import { resolvePcmOutputSampleRateHz } from "../pcm-sample-rates.js";
 import type { TtsProviderDefinition } from "../provider-definition.js";
 import type {
   TtsProvider,
@@ -25,11 +29,13 @@ import type {
 const log = getLogger("tts:fish-audio");
 
 /**
- * Sample rate requested when the caller wants PCM (the phone path).
- * Twilio media streams consume 8 kHz; without this Fish defaults WAV
- * to 44.1 kHz, which the media-stream transcoder would play ~5.5x slow.
+ * PCM/WAV sample rates the Fish Audio TTS API accepts. Per
+ * https://docs.fish.audio/api-reference/endpoint/openapi-v1/text-to-speech
+ * these are 8/16/24/32/44.1 kHz (22.05 and 48 kHz are not supported).
  */
-const TELEPHONY_SAMPLE_RATE_HZ = 8000;
+const SUPPORTED_PCM_SAMPLE_RATES_HZ = [
+  8_000, 16_000, 24_000, 32_000, 44_100,
+] as const;
 
 // ---------------------------------------------------------------------------
 // Error types
@@ -58,6 +64,7 @@ const FORMAT_CONTENT_TYPE: Record<string, string> = {
   mp3: "audio/mpeg",
   wav: "audio/wav",
   opus: "audio/opus",
+  pcm: "audio/pcm",
 };
 
 /**
@@ -80,115 +87,76 @@ function resolveReferenceId(
   return referenceId;
 }
 
+/** PCM rate resolver bound to the Fish-supported rate list (e.g. 48 kHz → 44.1 kHz). */
+const resolveFishPcmSampleRateHz = (request: TtsSynthesisRequest) =>
+  resolvePcmOutputSampleRateHz(request, SUPPORTED_PCM_SAMPLE_RATES_HZ);
+
 // ---------------------------------------------------------------------------
 // Provider implementation
 // ---------------------------------------------------------------------------
 
+async function performSynthesis(
+  request: TtsSynthesisRequest,
+  onChunk?: (chunk: Uint8Array) => void,
+): Promise<TtsSynthesisResult> {
+  const config = getConfig().services.tts.providers["fish-audio"];
+  const referenceId = resolveReferenceId(request, config);
+
+  // Fish Audio supports raw PCM (16-bit LE, no container) natively; the
+  // hinted sample rate is clamped to the nearest API-supported rate.
+  const pcmRequested = request.outputFormat === "pcm";
+  const effectiveFormat = pcmRequested ? "pcm" : config.format;
+  const sampleRateHz = resolveFishPcmSampleRateHz(request);
+
+  const effectiveConfig: FishAudioSynthesisConfig = {
+    ...config,
+    referenceId,
+    format: effectiveFormat,
+  };
+
+  const streaming = Boolean(onChunk);
+  log.info(
+    {
+      referenceId,
+      format: effectiveFormat,
+      streaming,
+      textLength: request.text.length,
+    },
+    "Starting Fish Audio TTS synthesis",
+  );
+
+  let audio: Buffer;
+  try {
+    audio = await synthesizeWithFishAudio(request.text, effectiveConfig, {
+      onChunk,
+      signal: request.signal,
+      sampleRate: sampleRateHz,
+    });
+  } catch (err) {
+    if (err instanceof Error && err.name === "AbortError") throw err;
+    throw new FishAudioTtsError(
+      "FISH_AUDIO_TTS_SYNTHESIS_FAILED",
+      `Fish Audio TTS ${streaming ? "streaming " : ""}synthesis failed: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+
+  const contentType = FORMAT_CONTENT_TYPE[effectiveFormat] ?? "audio/mpeg";
+
+  return { audio, contentType };
+}
+
 export function createFishAudioProvider(): TtsProvider {
   const capabilities: TtsProviderCapabilities = {
     supportsStreaming: true,
-    supportedFormats: ["mp3", "wav", "opus"],
+    supportedFormats: ["mp3", "wav", "opus", "pcm"],
   };
 
   return {
     id: "fish-audio",
     capabilities,
-
-    async synthesize(
-      request: TtsSynthesisRequest,
-    ): Promise<TtsSynthesisResult> {
-      const config = getConfig().services.tts.providers["fish-audio"];
-      const referenceId = resolveReferenceId(request, config);
-
-      // When PCM output is requested, override to WAV at 8 kHz. Fish Audio
-      // doesn't support raw PCM, but WAV gives us PCM in a container that
-      // audioBufferToFrames can extract; the explicit sample rate avoids
-      // Fish's 44.1 kHz WAV default.
-      const pcmRequested = request.outputFormat === "pcm";
-      const effectiveFormat = pcmRequested ? "wav" : config.format;
-
-      // Build an effective config with the resolved reference ID
-      // and the potentially overridden format.
-      const effectiveConfig: TtsFishAudioProviderConfig = {
-        ...config,
-        referenceId,
-        format: effectiveFormat,
-      };
-
-      log.info(
-        {
-          referenceId,
-          format: effectiveFormat,
-          textLength: request.text.length,
-        },
-        "Starting Fish Audio TTS synthesis",
-      );
-
-      let audio: Buffer;
-      try {
-        audio = await synthesizeWithFishAudio(request.text, effectiveConfig, {
-          signal: request.signal,
-          sampleRate: pcmRequested ? TELEPHONY_SAMPLE_RATE_HZ : undefined,
-        });
-      } catch (err) {
-        if (err instanceof Error && err.name === "AbortError") throw err;
-        throw new FishAudioTtsError(
-          "FISH_AUDIO_TTS_SYNTHESIS_FAILED",
-          `Fish Audio TTS synthesis failed: ${err instanceof Error ? err.message : String(err)}`,
-        );
-      }
-
-      const contentType = FORMAT_CONTENT_TYPE[effectiveFormat] ?? "audio/mpeg";
-
-      return { audio, contentType };
-    },
-
-    async synthesizeStream(
-      request: TtsSynthesisRequest,
-      onChunk: (chunk: Uint8Array) => void,
-    ): Promise<TtsSynthesisResult> {
-      const config = getConfig().services.tts.providers["fish-audio"];
-      const referenceId = resolveReferenceId(request, config);
-
-      // When PCM output is requested, override to WAV at 8 kHz (see
-      // synthesize above).
-      const pcmRequested = request.outputFormat === "pcm";
-      const effectiveFormat = pcmRequested ? "wav" : config.format;
-
-      const effectiveConfig: TtsFishAudioProviderConfig = {
-        ...config,
-        referenceId,
-        format: effectiveFormat,
-      };
-
-      log.info(
-        {
-          referenceId,
-          format: effectiveFormat,
-          textLength: request.text.length,
-        },
-        "Starting Fish Audio TTS streaming synthesis",
-      );
-
-      let audio: Buffer;
-      try {
-        audio = await synthesizeWithFishAudio(request.text, effectiveConfig, {
-          onChunk,
-          signal: request.signal,
-          sampleRate: pcmRequested ? TELEPHONY_SAMPLE_RATE_HZ : undefined,
-        });
-      } catch (err) {
-        if (err instanceof Error && err.name === "AbortError") throw err;
-        throw new FishAudioTtsError(
-          "FISH_AUDIO_TTS_SYNTHESIS_FAILED",
-          `Fish Audio TTS streaming synthesis failed: ${err instanceof Error ? err.message : String(err)}`,
-        );
-      }
-
-      const contentType = FORMAT_CONTENT_TYPE[effectiveFormat] ?? "audio/mpeg";
-
-      return { audio, contentType };
-    },
+    resolveOutputSampleRateHz: resolveFishPcmSampleRateHz,
+    synthesize: (request) => performSynthesis(request),
+    synthesizeStream: (request, onChunk) => performSynthesis(request, onChunk),
   };
 }
 
@@ -218,10 +186,9 @@ export const fishAudioTtsProviderDefinition: TtsProviderDefinition = {
   allowNativeFallback: true,
   capabilities: {
     supportsStreaming: true,
-    supportedFormats: ["mp3", "wav", "opus"],
+    supportedFormats: ["mp3", "wav", "opus", "pcm"],
   },
-  // The adapter substitutes WAV for the PCM hint (no raw PCM support).
-  mediaStreamPlayback: { outputFormat: "wav" },
+  mediaStreamPlayback: { outputFormat: "pcm" },
   secretRequirements: [
     {
       credentialStoreKey: "credential/fish-audio/api_key",

package/src/tts/providers/xai-provider.ts

@@ -1,9 +1,11 @@
 /**
  * xAI TTS provider adapter.
  *
- * Wraps the xAI REST text-to-speech API (`/v1/tts`) behind the uniform
- * {@link TtsProvider} interface. Reads the API key from the secure credential
- * store under `credential/xai/api_key` and the model configuration from the
+ * Wraps the xAI text-to-speech API behind the uniform {@link TtsProvider}
+ * interface: the REST endpoint (`/v1/tts`) for batch synthesis and the
+ * WebSocket endpoint (`wss://api.x.ai/v1/tts`) for chunk streaming. Reads
+ * the API key from the secure credential store under
+ * `credential/xai/api_key` and the model configuration from the
  * `services.tts.providers.xai` config section.
  */
 
@@ -12,13 +14,16 @@ import type { TtsXaiProviderConfig } from "../../config/schemas/tts.js";
 import { credentialKey } from "../../security/credential-key.js";
 import { getSecureKeyAsync } from "../../security/secure-keys.js";
 import { getLogger } from "../../util/logger.js";
+import { resolvePcmOutputSampleRateHz } from "../pcm-sample-rates.js";
 import type { TtsProviderDefinition } from "../provider-definition.js";
+import type { StreamReadTimeouts } from "../stream-read.js";
 import type {
   TtsProvider,
   TtsProviderCapabilities,
   TtsSynthesisRequest,
   TtsSynthesisResult,
 } from "../types.js";
+import { synthesizeOverXaiTtsSocket } from "./xai-tts-socket.js";
 
 const log = getLogger("tts:xai");
 
@@ -30,7 +35,9 @@ export type XaiTtsErrorCode =
   | "XAI_TTS_NO_API_KEY"
   | "XAI_TTS_HTTP_ERROR"
   | "XAI_TTS_EMPTY_RESPONSE"
-  | "XAI_TTS_REQUEST_FAILED";
+  | "XAI_TTS_REQUEST_FAILED"
+  | "XAI_TTS_STREAM_TIMEOUT"
+  | "XAI_TTS_STREAM_FAILED";
 
 export class XaiTtsError extends Error {
   readonly code: XaiTtsErrorCode;
@@ -50,6 +57,18 @@ export class XaiTtsError extends Error {
 
 const XAI_API_BASE = "https://api.x.ai";
 
+const XAI_WS_BASE = "wss://api.x.ai/v1/tts";
+
+/**
+ * Sample rates xAI TTS accepts (both REST and WS). Per
+ * https://docs.x.ai/developers/model-capabilities/audio/text-to-speech these
+ * are 8/16/22.05/24/44.1/48 kHz; the list intentionally matches the
+ * `services.tts.providers.xai.sampleRate` config validation.
+ */
+const SUPPORTED_PCM_SAMPLE_RATES_HZ = [
+  8_000, 16_000, 22_050, 24_000, 44_100, 48_000,
+] as const;
+
 /** Map from xAI codec names to MIME content types. */
 const FORMAT_CONTENT_TYPE: Record<string, string> = {
   mp3: "audio/mpeg",
@@ -73,15 +92,19 @@ interface XaiOutputParams {
   contentTypeKey: string;
 }
 
+/** PCM rate resolver bound to the xAI-supported rate list (e.g. 96 kHz → 48 kHz). */
+const resolveXaiPcmSampleRateHz = (request: TtsSynthesisRequest) =>
+  resolvePcmOutputSampleRateHz(request, SUPPORTED_PCM_SAMPLE_RATES_HZ);
+
 /**
  * Resolve the xAI output codec, sample rate, and bit rate based on the
  * synthesis request and provider config.
  *
  * **PCM path** (`outputFormat: "pcm"`):
  *   The media-stream transport needs raw headerless PCM for mu-law transcoding.
- *   We request `codec=pcm&sample_rate=16000` — matching the ElevenLabs /
- *   Deepgram 16 kHz PCM convention and the downstream `audioBufferToFrames`
- *   expectation (16 kHz -> 8 kHz downsample).
+ *   We request `codec=pcm` at the request's `sampleRateHz` hint clamped to the
+ *   nearest xAI-supported rate, defaulting to 16 kHz when no hint is given
+ *   (the shared no-hint convention across TTS providers).
  *
  * **MP3 path** (`config.format === "mp3"`):
  *   Uses the configured sample rate and bit rate.
@@ -93,10 +116,11 @@ function resolveOutputParams(
   request: TtsSynthesisRequest,
   config: TtsXaiProviderConfig,
 ): XaiOutputParams {
-  if (request.outputFormat === "pcm") {
+  const pcmSampleRateHz = resolveXaiPcmSampleRateHz(request);
+  if (pcmSampleRateHz != null) {
     return {
       codec: "pcm",
-      sample_rate: 16_000,
+      sample_rate: pcmSampleRateHz,
       contentTypeKey: "pcm",
     };
   }
@@ -125,27 +149,58 @@ function resolveVoiceId(
   return request.voiceId?.trim() || config.voiceId || "eve";
 }
 
-export function createXaiProvider(): TtsProvider {
+/** Resolve the xAI API key, throwing `XAI_TTS_NO_API_KEY` when unset. */
+async function requireApiKey(): Promise<string> {
+  const apiKey = await getSecureKeyAsync(credentialKey("xai", "api_key"));
+  if (!apiKey) {
+    throw new XaiTtsError(
+      "XAI_TTS_NO_API_KEY",
+      "xAI API key not configured. " +
+        "Add it via: assistant credentials set --service xai --field api_key <key>",
+    );
+  }
+  return apiKey;
+}
+
+/**
+ * Build the WebSocket synthesis URL. Mirrors the REST `output_format`
+ * payload (via {@link resolveOutputParams}) so both transports request
+ * identical audio.
+ */
+function buildStreamUrl(
+  request: TtsSynthesisRequest,
+  config: TtsXaiProviderConfig,
+  outputParams: XaiOutputParams,
+): string {
+  const params = new URLSearchParams({
+    language: config.language,
+    voice: resolveVoiceId(request, config),
+    codec: outputParams.codec,
+    sample_rate: String(outputParams.sample_rate),
+  });
+  if (outputParams.bit_rate != null) {
+    params.set("bit_rate", String(outputParams.bit_rate));
+  }
+  return `${XAI_WS_BASE}?${params.toString()}`;
+}
+
+export function createXaiProvider(
+  streamTimeouts: StreamReadTimeouts & { connectTimeoutMs?: number } = {},
+): TtsProvider {
   const capabilities: TtsProviderCapabilities = {
-    supportsStreaming: false,
-    supportedFormats: ["mp3", "wav"],
+    supportsStreaming: true,
+    supportedFormats: ["mp3", "wav", "pcm"],
   };
 
   return {
     id: "xai",
     capabilities,
+    resolveOutputSampleRateHz: resolveXaiPcmSampleRateHz,
 
     async synthesize(
       request: TtsSynthesisRequest,
     ): Promise<TtsSynthesisResult> {
-      const apiKey = await getSecureKeyAsync(credentialKey("xai", "api_key"));
-      if (!apiKey) {
-        throw new XaiTtsError(
-          "XAI_TTS_NO_API_KEY",
-          "xAI API key not configured. " +
-            "Add it via: assistant credentials set --service xai --field api_key <key>",
-        );
-      }
+      const apiKey = await requireApiKey();
 
       const config = getConfig().services.tts.providers.xai;
       const output = resolveOutputParams(request, config);
@@ -221,6 +276,60 @@ export function createXaiProvider(): TtsProvider {
         contentType,
       };
     },
+
+    async synthesizeStream(
+      request: TtsSynthesisRequest,
+      onChunk: (chunk: Uint8Array) => void,
+    ): Promise<TtsSynthesisResult> {
+      const apiKey = await requireApiKey();
+
+      const config = getConfig().services.tts.providers.xai;
+      const output = resolveOutputParams(request, config);
+      const url = buildStreamUrl(request, config, output);
+
+      log.info(
+        {
+          codec: output.codec,
+          sampleRate: output.sample_rate,
+          textLength: request.text.length,
+        },
+        "Starting xAI streaming TTS synthesis",
+      );
+
+      const audio = await synthesizeOverXaiTtsSocket({
+        url,
+        apiKey,
+        text: request.text,
+        onChunk,
+        signal: request.signal,
+        ...streamTimeouts,
+        makeTimeoutError: (timeoutMs) =>
+          new XaiTtsError(
+            "XAI_TTS_STREAM_TIMEOUT",
+            `xAI streaming TTS timed out after ${timeoutMs}ms`,
+          ),
+        makeStreamError: (detail) =>
+          new XaiTtsError(
+            "XAI_TTS_STREAM_FAILED",
+            `xAI streaming TTS failed: ${detail}`,
+          ),
+        makeEmptyError: () =>
+          new XaiTtsError(
+            "XAI_TTS_EMPTY_RESPONSE",
+            "xAI streaming TTS returned no audio",
+          ),
+      });
+
+      log.debug(
+        { bytes: audio.byteLength },
+        "xAI streaming TTS synthesis complete",
+      );
+
+      return {
+        audio,
+        contentType: FORMAT_CONTENT_TYPE[output.contentTypeKey] ?? "audio/mpeg",
+      };
+    },
   };
 }
 
@@ -248,8 +357,8 @@ export const xaiTtsProviderDefinition: TtsProviderDefinition = {
   callMode: "synthesized-play",
   allowNativeFallback: false,
   capabilities: {
-    supportsStreaming: false,
-    supportedFormats: ["mp3", "wav"],
+    supportsStreaming: true,
+    supportedFormats: ["mp3", "wav", "pcm"],
   },
   // The adapter honours the PCM hint via the `pcm` codec.
   mediaStreamPlayback: { outputFormat: "pcm" },