@tanstack/ai-grok 0.6.7 → 0.7.0

package/src/adapters/tts.ts

@@ -0,0 +1,260 @@
+import { BaseTTSAdapter } from '@tanstack/ai/adapters'
+import { arrayBufferToBase64, generateId, getGrokApiKeyFromEnv } from '../utils'
+import type { TTSOptions, TTSResult } from '@tanstack/ai'
+import type { GrokTTSModel } from '../model-meta'
+import type {
+  GrokTTSCodec,
+  GrokTTSProviderOptions,
+  GrokTTSVoice,
+} from '../audio/tts-provider-options'
+
+const DEFAULT_GROK_BASE_URL = 'https://api.x.ai/v1'
+
+/**
+ * Configuration for the Grok TTS adapter.
+ *
+ * Unlike chat/image/summarize adapters, TTS does not use the OpenAI SDK
+ * because xAI's `/v1/tts` endpoint is not OpenAI-compatible. This config
+ * is a minimal subset suitable for direct `fetch` calls.
+ */
+export interface GrokSpeechConfig {
+  apiKey: string
+  baseURL?: string
+  /** Additional headers to merge into every request (e.g., test IDs). */
+  defaultHeaders?: Record<string, string>
+}
+
+/**
+ * Grok Text-to-Speech Adapter.
+ *
+ * Talks to `POST {baseURL}/tts` per
+ * https://docs.x.ai/developers/model-capabilities/audio/text-to-speech
+ */
+export class GrokSpeechAdapter<
+  TModel extends GrokTTSModel,
+> extends BaseTTSAdapter<TModel, GrokTTSProviderOptions> {
+  readonly name = 'grok' as const
+
+  private readonly apiKey: string
+  private readonly baseURL: string
+  private readonly defaultHeaders: Record<string, string>
+
+  constructor(config: GrokSpeechConfig, model: TModel) {
+    super(model, config)
+    this.apiKey = config.apiKey
+    this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\/+$/, '')
+    this.defaultHeaders = config.defaultHeaders ?? {}
+  }
+
+  async generateSpeech(
+    options: TTSOptions<GrokTTSProviderOptions>,
+  ): Promise<TTSResult> {
+    const { logger } = options
+    const { model, text, voice, format, modelOptions } = options
+
+    logger.request(`activity=generateSpeech provider=grok model=${model}`, {
+      provider: 'grok',
+      model,
+    })
+
+    const { body, codec, sampleRateForContentType } = buildTTSRequestBody({
+      text,
+      voice,
+      format,
+      modelOptions,
+    })
+
+    try {
+      const response = await fetch(`${this.baseURL}/tts`, {
+        method: 'POST',
+        headers: {
+          // `defaultHeaders` first so the adapter's Authorization / Content-Type
+          // always win — otherwise a caller-supplied `Authorization` header
+          // could silently clobber the bearer token.
+          ...this.defaultHeaders,
+          Authorization: `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(body),
+      })
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new Error(
+          `Grok TTS request failed: ${response.status} ${errorText}`,
+        )
+      }
+
+      const arrayBuffer = await response.arrayBuffer()
+      const audio = arrayBufferToBase64(arrayBuffer)
+
+      return {
+        id: generateId(this.name),
+        model,
+        audio,
+        format: codec,
+        contentType: getContentType(codec, sampleRateForContentType),
+      }
+    } catch (error) {
+      logger.errors('grok.generateSpeech fatal', {
+        error,
+        source: 'grok.generateSpeech',
+      })
+      throw error
+    }
+  }
+}
+
+/**
+ * Build the JSON body for `POST /v1/tts`, resolving codec / sample-rate / voice
+ * defaults in one place.
+ *
+ * Returns the request `body`, the resolved `codec`, and the `sampleRateForContentType`
+ * used by the caller to label the response via `getContentType`.
+ */
+export function buildTTSRequestBody(options: {
+  text: string
+  voice: string | undefined
+  format: TTSOptions['format'] | undefined
+  modelOptions: GrokTTSProviderOptions | undefined
+}): {
+  body: Record<string, unknown>
+  codec: GrokTTSCodec
+  sampleRateForContentType: number
+} {
+  const { text, voice, format, modelOptions } = options
+
+  const codec = pickCodec(modelOptions?.codec, format)
+
+  // Only forward `sample_rate` when either:
+  //   - the caller explicitly set `modelOptions.sample_rate`, or
+  //   - the codec's Content-Type carries the rate (pcm → audio/L16;rate=…).
+  // For mp3/wav/opus/aac/flac we leave sample_rate unset so xAI's server
+  // default applies.
+  const callerSampleRate = modelOptions?.sample_rate
+  // Default sample rate documented in GrokTTSProviderOptions is 24000 Hz —
+  // used only when we MUST attach a rate to the contentType (pcm) and the
+  // caller didn't pick one.
+  const pcmDefault = 24000
+  const needsRateInContentType = codec === 'pcm'
+
+  const outputFormat: Record<string, unknown> = { codec }
+  if (callerSampleRate !== undefined) {
+    outputFormat.sample_rate = callerSampleRate
+  } else if (needsRateInContentType) {
+    outputFormat.sample_rate = pcmDefault
+  }
+  if (codec === 'mp3' && modelOptions?.bit_rate !== undefined) {
+    outputFormat.bit_rate = modelOptions.bit_rate
+  }
+
+  // pcm embeds the rate in `audio/L16;rate=…`; mulaw/alaw embed it in
+  // `audio/PCMU;rate=…` / `audio/PCMA;rate=…` when non-default. mp3/wav
+  // don't carry a rate parameter so the value is unused for those.
+  const sampleRateForContentType = callerSampleRate ?? pcmDefault
+
+  const body: Record<string, unknown> = {
+    text,
+    voice_id: (voice as GrokTTSVoice | undefined) ?? 'eve',
+    language: modelOptions?.language ?? 'en',
+    output_format: outputFormat,
+  }
+  if (modelOptions?.optimize_streaming_latency !== undefined) {
+    body.optimize_streaming_latency = modelOptions.optimize_streaming_latency
+  }
+  if (modelOptions?.text_normalization !== undefined) {
+    body.text_normalization = modelOptions.text_normalization
+  }
+
+  return { body, codec, sampleRateForContentType }
+}
+
+/**
+ * Maps the cross-provider `TTSOptions.format` onto Grok's supported codecs.
+ * `opus`, `aac`, and `flac` are not supported by xAI TTS (which only exposes
+ * mp3/wav/pcm/mulaw/alaw) — we fall back to mp3. An explicit
+ * `modelOptions.codec` always wins.
+ */
+function pickCodec(
+  codecOverride: GrokTTSCodec | undefined,
+  format: TTSOptions['format'] | undefined,
+): GrokTTSCodec {
+  if (codecOverride) return codecOverride
+  if (!format) return 'mp3'
+  switch (format) {
+    case 'mp3':
+    case 'wav':
+    case 'pcm':
+      return format
+    case 'flac':
+    case 'opus':
+    case 'aac':
+      return 'mp3'
+    default:
+      return 'mp3'
+  }
+}
+
+export function getContentType(
+  codec: GrokTTSCodec,
+  sampleRate: number,
+): string {
+  switch (codec) {
+    case 'mp3':
+      return 'audio/mpeg'
+    case 'wav':
+      return 'audio/wav'
+    case 'pcm':
+      // `audio/L16` requires a `rate` parameter per RFC 3551/3555.
+      return `audio/L16;rate=${sampleRate}`
+    case 'mulaw':
+      // `audio/basic` is 8 kHz mono by RFC 2046 registration. For non-8kHz
+      // streams xAI still produces mulaw-encoded bytes at the requested
+      // rate, but the registered MIME can't carry that rate — so we use
+      // the non-standard but commonly-supported `audio/PCMU;rate=…` (RFC 3551
+      // RTP payload name) whenever the caller asked for a rate other than
+      // 8000, and keep `audio/basic` for the standard 8kHz case.
+      return sampleRate === 8000
+        ? 'audio/basic'
+        : `audio/PCMU;rate=${sampleRate}`
+    case 'alaw':
+      return sampleRate === 8000
+        ? 'audio/x-alaw-basic'
+        : `audio/PCMA;rate=${sampleRate}`
+  }
+}
+
+/**
+ * Creates a Grok speech (TTS) adapter with an explicit API key.
+ *
+ * @example
+ * ```typescript
+ * const adapter = createGrokSpeech('grok-tts', 'xai-...')
+ * const result = await generateSpeech({
+ *   adapter,
+ *   text: 'Hello from Grok',
+ *   voice: 'eve',
+ * })
+ * ```
+ */
+export function createGrokSpeech<TModel extends GrokTTSModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<GrokSpeechConfig, 'apiKey'>,
+): GrokSpeechAdapter<TModel> {
+  return new GrokSpeechAdapter({ apiKey, ...config }, model)
+}
+
+/**
+ * Creates a Grok speech (TTS) adapter, reading the API key from
+ * `XAI_API_KEY` in the environment.
+ *
+ * @throws Error if `XAI_API_KEY` is not set.
+ */
+export function grokSpeech<TModel extends GrokTTSModel>(
+  model: TModel,
+  config?: Omit<GrokSpeechConfig, 'apiKey'>,
+): GrokSpeechAdapter<TModel> {
+  const apiKey = getGrokApiKeyFromEnv()
+  return createGrokSpeech(model, apiKey, config)
+}

package/src/audio/transcription-provider-options.ts

@@ -0,0 +1,54 @@
+/**
+ * Grok STT supported audio formats.
+ * See https://docs.x.ai/developers/rest-api-reference/inference/voice
+ */
+export type GrokSTTAudioFormat =
+  | 'pcm'
+  | 'mulaw'
+  | 'alaw'
+  | 'wav'
+  | 'mp3'
+  | 'ogg'
+  | 'opus'
+  | 'flac'
+  | 'aac'
+  | 'mp4'
+  | 'm4a'
+  | 'mkv'
+
+/**
+ * Provider-specific options for Grok transcription (`POST /v1/stt`).
+ */
+export interface GrokTranscriptionProviderOptions {
+  /**
+   * The format of the provided audio. Required for raw codecs (pcm, mulaw, alaw).
+   */
+  audio_format?: GrokSTTAudioFormat
+  /**
+   * Sample rate of the audio (Hz). Required for raw codecs.
+   */
+  sample_rate?: number
+  /**
+   * Apply inverse text normalization (e.g. "one hundred" → "100"). Requires
+   * `language` to be set on the core `TranscriptionOptions`.
+   *
+   * NOTE: xAI's STT API exposes this on the wire as `format` (a boolean
+   * toggle). We surface it under the clearer name
+   * `inverse_text_normalization` on the SDK, and translate to the wire name
+   * inside the adapter.
+   */
+  inverse_text_normalization?: boolean
+  /**
+   * Treat the audio as multichannel. When enabled, `channels` must also be set.
+   */
+  multichannel?: boolean
+  /**
+   * Channel count for multichannel raw audio (2–8).
+   */
+  channels?: number
+  /**
+   * Enable speaker diarization. When true, response words include a `speaker`
+   * field.
+   */
+  diarize?: boolean
+}

package/src/audio/tts-provider-options.ts

@@ -0,0 +1,44 @@
+/**
+ * Grok TTS voice options.
+ * See https://docs.x.ai/developers/model-capabilities/audio/text-to-speech
+ */
+export type GrokTTSVoice = 'eve' | 'ara' | 'rex' | 'sal' | 'leo'
+
+/**
+ * Grok TTS output audio codecs.
+ * Grok does NOT support opus or aac; those formats are mapped to mp3.
+ */
+export type GrokTTSCodec = 'mp3' | 'wav' | 'pcm' | 'mulaw' | 'alaw'
+
+/**
+ * Provider-specific options for Grok TTS (`POST /v1/tts`).
+ */
+export interface GrokTTSProviderOptions {
+  /**
+   * BCP-47 language code (e.g., `en`, `zh`, `pt-BR`) or `'auto'` for detection.
+   * Defaults to `'en'` when not provided.
+   */
+  language?: string
+  /**
+   * Audio codec. Overrides the `format` field on `TTSOptions` when set.
+   */
+  codec?: GrokTTSCodec
+  /**
+   * Sample rate in Hz. Valid values: 8000, 16000, 22050, 24000, 44100, 48000.
+   * Defaults to 24000.
+   */
+  sample_rate?: 8000 | 16000 | 22050 | 24000 | 44100 | 48000
+  /**
+   * Bit rate for MP3 output. Ignored for other codecs.
+   * Valid values: 32000, 64000, 96000, 128000, 192000. Defaults to 128000.
+   */
+  bit_rate?: 32000 | 64000 | 96000 | 128000 | 192000
+  /**
+   * Set to 1 for lower latency streaming; 0 (default) for normal quality.
+   */
+  optimize_streaming_latency?: 0 | 1
+  /**
+   * Enable text normalization. Defaults to false.
+   */
+  text_normalization?: boolean
+}

package/src/index.ts

@@ -33,6 +33,31 @@ export type {
   GrokImageModelProviderOptionsByName,
 } from './image/image-provider-options'
 
+// Speech (TTS) adapter - for text-to-speech
+export {
+  GrokSpeechAdapter,
+  createGrokSpeech,
+  grokSpeech,
+  type GrokSpeechConfig,
+} from './adapters/tts'
+export type {
+  GrokTTSProviderOptions,
+  GrokTTSVoice,
+  GrokTTSCodec,
+} from './audio/tts-provider-options'
+
+// Transcription adapter - for speech-to-text
+export {
+  GrokTranscriptionAdapter,
+  createGrokTranscription,
+  grokTranscription,
+  type GrokTranscriptionConfig,
+} from './adapters/transcription'
+export type {
+  GrokTranscriptionProviderOptions,
+  GrokSTTAudioFormat,
+} from './audio/transcription-provider-options'
+
 // ============================================================================
 // Type Exports
 // ============================================================================
@@ -45,8 +70,17 @@ export type {
   ResolveInputModalities,
   GrokChatModel,
   GrokImageModel,
+  GrokTTSModel,
+  GrokTranscriptionModel,
+  GrokRealtimeModel,
+} from './model-meta'
+export {
+  GROK_CHAT_MODELS,
+  GROK_IMAGE_MODELS,
+  GROK_TTS_MODELS,
+  GROK_TRANSCRIPTION_MODELS,
+  GROK_REALTIME_MODELS,
 } from './model-meta'
-export { GROK_CHAT_MODELS, GROK_IMAGE_MODELS } from './model-meta'
 export type {
   GrokTextMetadata,
   GrokImageMetadata,
@@ -55,3 +89,18 @@ export type {
   GrokDocumentMetadata,
   GrokMessageMetadataByModality,
 } from './message-types'
+
+// ============================================================================
+// Realtime (Voice Agent) Adapters
+// ============================================================================
+
+export { grokRealtimeToken, grokRealtime } from './realtime/index'
+
+export type {
+  GrokRealtimeVoice,
+  GrokRealtimeTokenOptions,
+  GrokRealtimeOptions,
+  GrokTurnDetection,
+  GrokSemanticVADConfig,
+  GrokServerVADConfig,
+} from './realtime/index'

package/src/model-meta.ts

@@ -283,8 +283,62 @@ export const GROK_CHAT_MODELS = [
  */
 export const GROK_IMAGE_MODELS = [GROK_2_IMAGE.name] as const
 
+// xAI's `/v1/tts` endpoint is endpoint-addressed and does not take a `model`
+// parameter. This synthetic identifier satisfies the SDK's `TTSOptions.model`
+// contract and provides a stable value for logging and fixture matching.
+const GROK_TTS = {
+  name: 'grok-tts',
+  supports: {
+    input: ['text'],
+    output: ['audio'],
+  },
+} as const satisfies ModelMeta
+
+// xAI's `/v1/stt` endpoint is endpoint-addressed and does not take a `model`
+// parameter. This synthetic identifier satisfies the SDK's
+// `TranscriptionOptions.model` contract.
+const GROK_STT = {
+  name: 'grok-stt',
+  supports: {
+    input: ['audio'],
+    output: ['text'],
+  },
+} as const satisfies ModelMeta
+
+const GROK_VOICE_FAST_1 = {
+  name: 'grok-voice-fast-1.0',
+  supports: {
+    input: ['audio', 'text'],
+    output: ['audio', 'text'],
+    capabilities: ['tool_calling'],
+    tools: [] as const,
+  },
+} as const satisfies ModelMeta
+
+const GROK_VOICE_THINK_FAST_1 = {
+  name: 'grok-voice-think-fast-1.0',
+  supports: {
+    input: ['audio', 'text'],
+    output: ['audio', 'text'],
+    capabilities: ['reasoning', 'tool_calling'],
+    tools: [] as const,
+  },
+} as const satisfies ModelMeta
+
+export const GROK_TTS_MODELS = [GROK_TTS.name] as const
+
+export const GROK_TRANSCRIPTION_MODELS = [GROK_STT.name] as const
+
+export const GROK_REALTIME_MODELS = [
+  GROK_VOICE_FAST_1.name,
+  GROK_VOICE_THINK_FAST_1.name,
+] as const
+
 export type GrokChatModel = (typeof GROK_CHAT_MODELS)[number]
 export type GrokImageModel = (typeof GROK_IMAGE_MODELS)[number]
+export type GrokTTSModel = (typeof GROK_TTS_MODELS)[number]
+export type GrokTranscriptionModel = (typeof GROK_TRANSCRIPTION_MODELS)[number]
+export type GrokRealtimeModel = (typeof GROK_REALTIME_MODELS)[number]
 
 /**
  * Type-only map from Grok chat model name to its supported input modalities.