@vellumai/assistant 0.6.4 → 0.6.5

package/src/tts/__tests__/provider-adapters.test.ts

@@ -35,6 +35,14 @@ let mockDeepgramConfig = {
   format: "mp3" as "mp3" | "wav" | "opus",
 };
 
+let mockXaiConfig = {
+  voiceId: "eve",
+  language: "auto",
+  format: "mp3" as "mp3" | "wav",
+  sampleRate: 24000,
+  bitRate: 128000,
+};
+
 mock.module("../../config/loader.js", () => ({
   getConfig: () => ({
     services: {
@@ -43,6 +51,7 @@ mock.module("../../config/loader.js", () => ({
           elevenlabs: mockElevenLabsConfig,
           "fish-audio": mockFishAudioConfig,
           deepgram: mockDeepgramConfig,
+          xai: mockXaiConfig,
         },
       },
     },
@@ -53,9 +62,13 @@ mock.module("../../config/loader.js", () => ({
 
 let mockApiKey: string | null = "test-elevenlabs-api-key";
 let mockDeepgramApiKey: string | null = "test-deepgram-api-key";
+let mockXaiApiKey: string | null = "test-xai-api-key";
 
 mock.module("../../security/secure-keys.js", () => ({
-  getSecureKeyAsync: async () => mockApiKey,
+  getSecureKeyAsync: async (key?: string) => {
+    if (key === "credential/xai/api_key") return mockXaiApiKey;
+    return mockApiKey;
+  },
   getProviderKeyAsync: async (provider: string) => {
     if (provider === "deepgram") return mockDeepgramApiKey;
     return mockApiKey;
@@ -110,6 +123,7 @@ import {
   _resetBuiltinRegistration,
   registerBuiltinTtsProviders,
 } from "../providers/register-builtins.js";
+import { createXaiProvider, XaiTtsError } from "../providers/xai-provider.js";
 import type { TtsSynthesisRequest } from "../types.js";
 
 // ---------------------------------------------------------------------------
@@ -141,6 +155,14 @@ beforeEach(() => {
     model: "aura-asteria-en",
     format: "mp3",
   };
+  mockXaiApiKey = "test-xai-api-key";
+  mockXaiConfig = {
+    voiceId: "eve",
+    language: "auto",
+    format: "mp3",
+    sampleRate: 24000,
+    bitRate: 128000,
+  };
   mockSynthesizeWithFishAudio.mockClear();
 });
 
@@ -759,32 +781,237 @@ describe("Deepgram TTS provider adapter", () => {
   });
 });
 
+// ===========================================================================
+// xAI TTS provider adapter
+// ===========================================================================
+
+describe("xAI TTS provider adapter", () => {
+  // -- Interface conformance -----------------------------------------------
+
+  test("has correct provider ID", () => {
+    const provider = createXaiProvider();
+    expect(provider.id).toBe("xai");
+  });
+
+  test("advertises mp3 and wav format support without streaming", () => {
+    const provider = createXaiProvider();
+    expect(provider.capabilities).toEqual({
+      supportsStreaming: false,
+      supportedFormats: ["mp3", "wav"],
+    });
+  });
+
+  // -- Request mapping -----------------------------------------------------
+
+  test("synthesize posts to /v1/tts with correct auth and default body", async () => {
+    const audioPayload = new Uint8Array([0x49, 0x44, 0x33]);
+    let capturedUrl = "";
+    let capturedHeaders: Headers | null = null;
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (input: RequestInfo | URL, init?: RequestInit) => {
+        capturedUrl = typeof input === "string" ? input : input.toString();
+        capturedHeaders = new Headers(init?.headers);
+        capturedBody = init?.body as string;
+        return new Response(audioPayload, {
+          status: 200,
+          headers: { "Content-Type": "audio/mpeg" },
+        });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createXaiProvider();
+    await provider.synthesize(makeRequest());
+
+    expect(capturedUrl).toBe("https://api.x.ai/v1/tts");
+    expect(capturedHeaders!.get("Authorization")).toBe(
+      "Bearer test-xai-api-key",
+    );
+    expect(capturedHeaders!.get("Content-Type")).toBe("application/json");
+
+    const body = JSON.parse(capturedBody);
+    expect(body.text).toBe("Hello world");
+    expect(body.voice_id).toBe("eve");
+    expect(body.language).toBe("auto");
+    expect(body.output_format).toEqual({
+      codec: "mp3",
+      sample_rate: 24000,
+      bit_rate: 128000,
+    });
+  });
+
+  test("request voiceId overrides config voiceId", async () => {
+    const audioPayload = new Uint8Array([0x49, 0x44, 0x33]);
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(audioPayload, { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createXaiProvider();
+    await provider.synthesize(makeRequest({ voiceId: "rex" }));
+
+    const body = JSON.parse(capturedBody);
+    expect(body.voice_id).toBe("rex");
+  });
+
+  test("uses configured voiceId when request has none", async () => {
+    mockXaiConfig.voiceId = "ara";
+    const audioPayload = new Uint8Array([0x49, 0x44, 0x33]);
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(audioPayload, { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createXaiProvider();
+    await provider.synthesize(makeRequest());
+
+    const body = JSON.parse(capturedBody);
+    expect(body.voice_id).toBe("ara");
+  });
+
+  test("wav config format produces codec=wav without bit_rate", async () => {
+    mockXaiConfig.format = "wav";
+    const audioPayload = new Uint8Array([0x52, 0x49, 0x46, 0x46]);
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(audioPayload, { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createXaiProvider();
+    const result = await provider.synthesize(makeRequest());
+
+    const body = JSON.parse(capturedBody);
+    expect(body.output_format).toEqual({
+      codec: "wav",
+      sample_rate: 24000,
+    });
+    expect(body.output_format.bit_rate).toBeUndefined();
+    expect(result.contentType).toBe("audio/wav");
+  });
+
+  test("outputFormat=pcm uses codec=pcm and 16 kHz sample rate", async () => {
+    const audioPayload = new Uint8Array([0x00, 0x01, 0x02]);
+    let capturedBody = "";
+
+    globalThis.fetch = mock(
+      async (_input: RequestInfo | URL, init?: RequestInit) => {
+        capturedBody = init?.body as string;
+        return new Response(audioPayload, { status: 200 });
+      },
+    ) as unknown as typeof globalThis.fetch;
+
+    const provider = createXaiProvider();
+    const result = await provider.synthesize(
+      makeRequest({ outputFormat: "pcm" }),
+    );
+
+    const body = JSON.parse(capturedBody);
+    expect(body.output_format).toEqual({
+      codec: "pcm",
+      sample_rate: 16000,
+    });
+    expect(body.output_format.bit_rate).toBeUndefined();
+    expect(result.contentType).toBe("audio/pcm");
+  });
+
+  // -- Content type / format -----------------------------------------------
+
+  test("returns audio/mpeg content type for mp3 format", async () => {
+    const audioPayload = new Uint8Array([0x49, 0x44, 0x33]);
+    mockFetchReturning(audioPayload);
+
+    const provider = createXaiProvider();
+    const result = await provider.synthesize(makeRequest());
+
+    expect(result.contentType).toBe("audio/mpeg");
+    expect(result.audio.byteLength).toBeGreaterThan(0);
+  });
+
+  // -- Required config validation ------------------------------------------
+
+  test("throws XAI_TTS_NO_API_KEY when API key is missing", async () => {
+    mockXaiApiKey = null;
+
+    const provider = createXaiProvider();
+
+    try {
+      await provider.synthesize(makeRequest());
+      throw new Error("Expected synthesize to throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(XaiTtsError);
+      expect((err as XaiTtsError).code).toBe("XAI_TTS_NO_API_KEY");
+      expect((err as XaiTtsError).message).toContain("API key not configured");
+    }
+  });
+
+  // -- Error handling ------------------------------------------------------
+
+  test("throws XAI_TTS_HTTP_ERROR on non-200 response", async () => {
+    mockFetchError(401, "Unauthorized");
+
+    const provider = createXaiProvider();
+
+    try {
+      await provider.synthesize(makeRequest());
+      throw new Error("Expected synthesize to throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(XaiTtsError);
+      expect((err as XaiTtsError).code).toBe("XAI_TTS_HTTP_ERROR");
+      expect((err as XaiTtsError).statusCode).toBe(401);
+      expect((err as XaiTtsError).message).toContain("Unauthorized");
+    }
+  });
+
+  test("throws XAI_TTS_EMPTY_RESPONSE on empty audio body", async () => {
+    mockFetchReturning(new Uint8Array(0));
+
+    const provider = createXaiProvider();
+
+    try {
+      await provider.synthesize(makeRequest());
+      throw new Error("Expected synthesize to throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(XaiTtsError);
+      expect((err as XaiTtsError).code).toBe("XAI_TTS_EMPTY_RESPONSE");
+    }
+  });
+});
+
 // ===========================================================================
 // Built-in registration
 // ===========================================================================
 
 describe("registerBuiltinTtsProviders", () => {
-  test("registers elevenlabs, fish-audio, and deepgram providers", () => {
+  test("registers every catalog provider ID", () => {
     registerBuiltinTtsProviders();
 
     const providers = listTtsProviders();
     const ids = providers.map((p) => p.id);
-    expect(ids).toContain("elevenlabs");
-    expect(ids).toContain("fish-audio");
-    expect(ids).toContain("deepgram");
+    for (const id of listCatalogProviderIds()) {
+      expect(ids).toContain(id);
+    }
   });
 
   test("providers are discoverable via getTtsProvider after registration", () => {
     registerBuiltinTtsProviders();
 
-    const el = getTtsProvider("elevenlabs");
-    expect(el.id).toBe("elevenlabs");
-
-    const fa = getTtsProvider("fish-audio");
-    expect(fa.id).toBe("fish-audio");
-
-    const dg = getTtsProvider("deepgram");
-    expect(dg.id).toBe("deepgram");
+    for (const id of listCatalogProviderIds()) {
+      const provider = getTtsProvider(id);
+      expect(provider.id).toBe(id);
+    }
   });
 
   test("idempotent — calling twice does not throw", () => {

package/src/tts/provider-catalog.ts

@@ -158,6 +158,24 @@ const CATALOG: readonly TtsProviderCatalogEntry[] = [
       },
     ],
   },
+  {
+    id: "xai",
+    displayName: "xAI",
+    callMode: "synthesized-play",
+    allowNativeFallback: false,
+    capabilities: {
+      supportsStreaming: false,
+      supportedFormats: ["mp3", "wav"],
+    },
+    secretRequirements: [
+      {
+        credentialStoreKey: "credential/xai/api_key",
+        displayName: "xAI API Key",
+        setCommand:
+          "assistant credentials set --service xai --field api_key <key>",
+      },
+    ],
+  },
 ] as const;
 
 /** Index for O(1) lookup by provider ID. */

package/src/tts/providers/index.ts

@@ -13,6 +13,7 @@ import type { TtsProvider, TtsProviderId } from "../types.js";
 import { createDeepgramProvider } from "./deepgram-provider.js";
 import { createElevenLabsProvider } from "./elevenlabs-provider.js";
 import { createFishAudioProvider } from "./fish-audio-provider.js";
+import { createXaiProvider } from "./xai-provider.js";
 
 // ---------------------------------------------------------------------------
 // Factory type
@@ -39,4 +40,5 @@ export const providerFactories: ReadonlyMap<TtsProviderId, TtsProviderFactory> =
     ["elevenlabs", createElevenLabsProvider],
     ["fish-audio", createFishAudioProvider],
     ["deepgram", createDeepgramProvider],
+    ["xai", createXaiProvider],
   ]);

package/src/tts/providers/xai-provider.ts

@@ -0,0 +1,224 @@
+/**
+ * xAI TTS provider adapter.
+ *
+ * Wraps the xAI REST text-to-speech API (`/v1/tts`) behind the uniform
+ * {@link TtsProvider} interface. Reads the API key from the secure credential
+ * store under `credential/xai/api_key` and the model configuration from the
+ * `services.tts.providers.xai` config section.
+ */
+
+import { getConfig } from "../../config/loader.js";
+import type { TtsXaiProviderConfig } from "../../config/schemas/tts.js";
+import { credentialKey } from "../../security/credential-key.js";
+import { getSecureKeyAsync } from "../../security/secure-keys.js";
+import { getLogger } from "../../util/logger.js";
+import type {
+  TtsProvider,
+  TtsProviderCapabilities,
+  TtsSynthesisRequest,
+  TtsSynthesisResult,
+} from "../types.js";
+
+const log = getLogger("tts:xai");
+
+// ---------------------------------------------------------------------------
+// Error types
+// ---------------------------------------------------------------------------
+
+export type XaiTtsErrorCode =
+  | "XAI_TTS_NO_API_KEY"
+  | "XAI_TTS_HTTP_ERROR"
+  | "XAI_TTS_EMPTY_RESPONSE"
+  | "XAI_TTS_REQUEST_FAILED";
+
+export class XaiTtsError extends Error {
+  readonly code: XaiTtsErrorCode;
+  readonly statusCode?: number;
+
+  constructor(code: XaiTtsErrorCode, message: string, statusCode?: number) {
+    super(message);
+    this.name = "XaiTtsError";
+    this.code = code;
+    this.statusCode = statusCode;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const XAI_API_BASE = "https://api.x.ai";
+
+/** Map from xAI codec names to MIME content types. */
+const FORMAT_CONTENT_TYPE: Record<string, string> = {
+  mp3: "audio/mpeg",
+  wav: "audio/wav",
+  pcm: "audio/pcm",
+};
+
+// ---------------------------------------------------------------------------
+// Provider implementation
+// ---------------------------------------------------------------------------
+
+/** Parameters for xAI's `/v1/tts` output_format payload. */
+interface XaiOutputParams {
+  /** xAI codec name (`mp3`, `wav`, or `pcm`). */
+  codec: string;
+  /** Sample rate in Hz. */
+  sample_rate: number;
+  /** MP3 bit rate. Omitted for non-MP3 codecs. */
+  bit_rate?: number;
+  /** Content-type key for the FORMAT_CONTENT_TYPE lookup. */
+  contentTypeKey: string;
+}
+
+/**
+ * Resolve the xAI output codec, sample rate, and bit rate based on the
+ * synthesis request and provider config.
+ *
+ * **PCM path** (`outputFormat: "pcm"`):
+ *   The media-stream transport needs raw headerless PCM for mu-law transcoding.
+ *   We request `codec=pcm&sample_rate=16000` — matching the ElevenLabs /
+ *   Deepgram 16 kHz PCM convention and the downstream `audioBufferToFrames`
+ *   expectation (16 kHz -> 8 kHz downsample).
+ *
+ * **MP3 path** (`config.format === "mp3"`):
+ *   Uses the configured sample rate and bit rate.
+ *
+ * **WAV path** (`config.format === "wav"`):
+ *   Uses the configured sample rate; bit rate is not meaningful for WAV.
+ */
+function resolveOutputParams(
+  request: TtsSynthesisRequest,
+  config: TtsXaiProviderConfig,
+): XaiOutputParams {
+  if (request.outputFormat === "pcm") {
+    return {
+      codec: "pcm",
+      sample_rate: 16_000,
+      contentTypeKey: "pcm",
+    };
+  }
+
+  if (config.format === "mp3") {
+    return {
+      codec: "mp3",
+      sample_rate: config.sampleRate,
+      bit_rate: config.bitRate,
+      contentTypeKey: "mp3",
+    };
+  }
+
+  return {
+    codec: "wav",
+    sample_rate: config.sampleRate,
+    contentTypeKey: "wav",
+  };
+}
+
+/** Resolve the voice ID: request override > config > default. */
+function resolveVoiceId(
+  request: TtsSynthesisRequest,
+  config: TtsXaiProviderConfig,
+): string {
+  return request.voiceId?.trim() || config.voiceId || "eve";
+}
+
+export function createXaiProvider(): TtsProvider {
+  const capabilities: TtsProviderCapabilities = {
+    supportsStreaming: false,
+    supportedFormats: ["mp3", "wav"],
+  };
+
+  return {
+    id: "xai",
+    capabilities,
+
+    async synthesize(
+      request: TtsSynthesisRequest,
+    ): Promise<TtsSynthesisResult> {
+      const apiKey = await getSecureKeyAsync(credentialKey("xai", "api_key"));
+      if (!apiKey) {
+        throw new XaiTtsError(
+          "XAI_TTS_NO_API_KEY",
+          "xAI API key not configured. " +
+            "Add it via: assistant credentials set --service xai --field api_key <key>",
+        );
+      }
+
+      const config = getConfig().services.tts.providers.xai;
+      const output = resolveOutputParams(request, config);
+      const voiceId = resolveVoiceId(request, config);
+
+      const body = {
+        text: request.text,
+        voice_id: voiceId,
+        language: config.language,
+        output_format: {
+          codec: output.codec,
+          sample_rate: output.sample_rate,
+          ...(output.bit_rate ? { bit_rate: output.bit_rate } : {}),
+        },
+      };
+
+      log.info(
+        {
+          voiceId,
+          codec: output.codec,
+          sampleRate: output.sample_rate,
+          textLength: request.text.length,
+        },
+        "Starting xAI TTS synthesis",
+      );
+
+      let response: Response;
+      try {
+        response = await fetch(`${XAI_API_BASE}/v1/tts`, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${apiKey}`,
+          },
+          body: JSON.stringify(body),
+          signal: request.signal,
+        });
+      } catch (err) {
+        if (err instanceof Error && err.name === "AbortError") throw err;
+        throw new XaiTtsError(
+          "XAI_TTS_REQUEST_FAILED",
+          `xAI TTS request failed: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+
+      if (!response.ok) {
+        const errorText = await response.text().catch(() => "");
+        throw new XaiTtsError(
+          "XAI_TTS_HTTP_ERROR",
+          `xAI TTS returned ${response.status}: ${errorText}`,
+          response.status,
+        );
+      }
+
+      const arrayBuffer = await response.arrayBuffer();
+      if (arrayBuffer.byteLength === 0) {
+        throw new XaiTtsError(
+          "XAI_TTS_EMPTY_RESPONSE",
+          "xAI TTS returned an empty audio response",
+        );
+      }
+
+      const contentType =
+        FORMAT_CONTENT_TYPE[output.contentTypeKey] ?? "audio/mpeg";
+
+      log.debug(
+        { bytes: arrayBuffer.byteLength },
+        "xAI TTS synthesis complete",
+      );
+
+      return {
+        audio: Buffer.from(arrayBuffer),
+        contentType,
+      };
+    },
+  };
+}

package/src/tts/types.ts

@@ -20,6 +20,7 @@ export type TtsProviderId =
   | "elevenlabs"
   | "fish-audio"
   | "deepgram"
+  | "xai"
   | (string & {});
 
 // ---------------------------------------------------------------------------
@@ -99,6 +100,35 @@ export interface TtsSynthesisResult {
   contentType: string;
 }
 
+// ---------------------------------------------------------------------------
+// Alignment / viseme events
+// ---------------------------------------------------------------------------
+
+/**
+ * Per-phoneme alignment event emitted by TTS providers that expose
+ * character- or phoneme-level alignment metadata alongside the audio
+ * stream (e.g. ElevenLabs Turbo with alignment).
+ *
+ * Consumers (e.g. the Meet avatar lip-sync path) map these events to
+ * blendshape weights at the rendered timestamp. Providers that do not
+ * expose alignment metadata simply never invoke the callback; the
+ * caller is expected to fall back to an amplitude-envelope approximation
+ * derived from the PCM stream.
+ */
+export interface TtsAlignmentEvent {
+  /**
+   * Phoneme label or character — free-form string that the consumer maps
+   * to renderer-specific blendshape weights. Providers typically emit
+   * IPA-style phoneme labels, viseme codes, or individual characters
+   * depending on their alignment granularity.
+   */
+  phoneme: string;
+  /** Normalized intensity in the range [0, 1]. */
+  weight: number;
+  /** Milliseconds from the start of the synthesized utterance. */
+  timestamp: number;
+}
+
 // ---------------------------------------------------------------------------
 // Provider capabilities
 // ---------------------------------------------------------------------------
@@ -110,6 +140,15 @@ export interface TtsProviderCapabilities {
 
   /** Audio formats the provider can produce (e.g. `["mp3", "wav", "opus"]`). */
   supportedFormats: string[];
+
+  /**
+   * Whether the provider can emit {@link TtsAlignmentEvent}s via the optional
+   * `onAlignment` callback of `synthesizeStream`. Consumers use this to pick
+   * between a viseme-driven lip-sync path and an RMS-amplitude fallback.
+   *
+   * Optional for backwards compatibility — treat `undefined` as `false`.
+   */
+  alignment?: boolean;
 }
 
 // ---------------------------------------------------------------------------
@@ -145,9 +184,16 @@ export interface TtsProvider {
    * The `onChunk` callback is invoked with each audio chunk as it arrives
    * from the upstream provider. The returned promise resolves with the
    * complete concatenated result once all chunks have been delivered.
+   *
+   * Providers that advertise `capabilities.alignment === true` also invoke
+   * the optional `onAlignment` callback with per-phoneme alignment events
+   * interleaved with the audio chunks. Providers that don't support
+   * alignment simply never call it; callers must tolerate a silent
+   * channel and fall back to amplitude-based heuristics.
    */
   synthesizeStream?(
     request: TtsSynthesisRequest,
     onChunk: (chunk: Uint8Array) => void,
+    onAlignment?: (event: TtsAlignmentEvent) => void,
   ): Promise<TtsSynthesisResult>;
 }