@tanstack/ai-grok 0.6.8 → 0.7.1

package/src/adapters/image.ts

@@ -49,7 +49,7 @@ export class GrokImageAdapter<
   private client: OpenAI_SDK
 
   constructor(config: GrokImageConfig, model: TModel) {
-    super({}, model)
+    super(model, {})
     this.client = createGrokClient(config)
   }
 
@@ -92,12 +92,14 @@ export class GrokImageAdapter<
   ): OpenAI_SDK.Images.ImageGenerateParams {
     const { model, prompt, numberOfImages, size, modelOptions } = options
 
+    // Spread modelOptions FIRST so explicit args (model, prompt, n, size) win
+    // and user-supplied modelOptions cannot silently override them.
     return {
+      ...modelOptions,
       model,
       prompt,
       n: numberOfImages ?? 1,
       size: size as OpenAI_SDK.Images.ImageGenerateParams['size'],
-      ...modelOptions,
     }
   }
 
@@ -105,11 +107,18 @@ export class GrokImageAdapter<
     model: string,
     response: OpenAI_SDK.Images.ImagesResponse,
   ): ImageGenerationResult {
-    const images: Array<GeneratedImage> = (response.data ?? []).map((item) => ({
-      b64Json: item.b64_json,
-      url: item.url,
-      revisedPrompt: item.revised_prompt,
-    }))
+    const images: Array<GeneratedImage> = (response.data ?? []).flatMap(
+      (item): Array<GeneratedImage> => {
+        const revisedPrompt = item.revised_prompt
+        if (item.b64_json) {
+          return [{ b64Json: item.b64_json, revisedPrompt }]
+        }
+        if (item.url) {
+          return [{ url: item.url, revisedPrompt }]
+        }
+        return []
+      },
+    )
 
     return {
       id: generateId(this.name),

package/src/adapters/transcription.ts

@@ -0,0 +1,233 @@
+import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters'
+import { generateId, getGrokApiKeyFromEnv, toAudioFile } from '../utils'
+import type {
+  TranscriptionOptions,
+  TranscriptionResult,
+  TranscriptionWord,
+} from '@tanstack/ai'
+import type { GrokTranscriptionModel } from '../model-meta'
+import type { GrokTranscriptionProviderOptions } from '../audio/transcription-provider-options'
+
+/**
+ * Grok-specific extension of `TranscriptionWord` that surfaces the extra
+ * fields xAI returns when diarization / confidence are enabled. The base
+ * cross-provider `TranscriptionWord` contract doesn't include these, so
+ * callers who know they're using Grok can narrow with:
+ *
+ * ```ts
+ * const words = result.words as Array<GrokTranscriptionWord> | undefined
+ * ```
+ */
+export interface GrokTranscriptionWord extends TranscriptionWord {
+  /** Model confidence for the word, when xAI returns one. */
+  confidence?: number
+  /** Speaker index, populated when `modelOptions.diarize === true`. */
+  speaker?: number
+}
+
+const DEFAULT_GROK_BASE_URL = 'https://api.x.ai/v1'
+
+/**
+ * Configuration for the Grok transcription adapter.
+ *
+ * Uses direct `fetch` rather than the OpenAI SDK because xAI's `/v1/stt`
+ * endpoint is not OpenAI-compatible.
+ */
+export interface GrokTranscriptionConfig {
+  apiKey: string
+  baseURL?: string
+  /** Additional headers to merge into every request (e.g., test IDs). */
+  defaultHeaders?: Record<string, string>
+}
+
+/**
+ * xAI STT response shape from `POST /v1/stt`.
+ * Grok returns word-level timestamps only; no segment array.
+ */
+interface GrokSTTWord {
+  text: string
+  start: number
+  end: number
+  confidence?: number
+  speaker?: number
+}
+
+interface GrokSTTResponse {
+  text: string
+  language?: string
+  duration?: number
+  words?: Array<GrokSTTWord>
+  channels?: Array<unknown>
+}
+
+/**
+ * Grok Speech-to-Text Adapter.
+ *
+ * Talks to `POST {baseURL}/stt` per
+ * https://docs.x.ai/developers/rest-api-reference/inference/voice
+ */
+export class GrokTranscriptionAdapter<
+  TModel extends GrokTranscriptionModel,
+> extends BaseTranscriptionAdapter<TModel, GrokTranscriptionProviderOptions> {
+  readonly name = 'grok' as const
+
+  private readonly apiKey: string
+  private readonly baseURL: string
+  private readonly defaultHeaders: Record<string, string>
+
+  constructor(config: GrokTranscriptionConfig, model: TModel) {
+    super(model, config)
+    this.apiKey = config.apiKey
+    this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\/+$/, '')
+    this.defaultHeaders = config.defaultHeaders ?? {}
+  }
+
+  async transcribe(
+    options: TranscriptionOptions<GrokTranscriptionProviderOptions>,
+  ): Promise<TranscriptionResult> {
+    const { logger } = options
+    const { model, audio, language, modelOptions } = options
+
+    logger.request(
+      `activity=generateTranscription provider=grok model=${model}`,
+      { provider: 'grok', model },
+    )
+
+    const file = toAudioFile(audio, modelOptions?.audio_format)
+    const form = buildTranscriptionFormData({ file, language, modelOptions })
+
+    try {
+      const response = await fetch(`${this.baseURL}/stt`, {
+        method: 'POST',
+        headers: {
+          // `defaultHeaders` first so Authorization always wins.
+          ...this.defaultHeaders,
+          Authorization: `Bearer ${this.apiKey}`,
+        },
+        body: form,
+      })
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new Error(
+          `Grok transcription request failed: ${response.status} ${errorText}`,
+        )
+      }
+
+      const data = (await response.json()) as GrokSTTResponse
+
+      const words: Array<TranscriptionWord> | undefined = data.words?.map(
+        (w) => {
+          // Construct a GrokTranscriptionWord so that `confidence` and
+          // `speaker` (when xAI returns them under `diarize` / confidence
+          // mode) are preserved on the result. The returned array is typed
+          // as `Array<TranscriptionWord>` per the cross-provider contract;
+          // callers who want the extras narrow via `as Array<GrokTranscriptionWord>`.
+          const tw: GrokTranscriptionWord = {
+            word: w.text,
+            start: w.start,
+            end: w.end,
+          }
+          if (w.confidence !== undefined) tw.confidence = w.confidence
+          if (w.speaker !== undefined) tw.speaker = w.speaker
+          return tw
+        },
+      )
+
+      return {
+        id: generateId(this.name),
+        model,
+        text: data.text,
+        language: data.language ?? language,
+        duration: data.duration,
+        words,
+      }
+    } catch (error) {
+      logger.errors('grok.transcribe fatal', {
+        error,
+        source: 'grok.transcribe',
+      })
+      throw error
+    }
+  }
+}
+
+/**
+ * Build the multipart/form-data body for `POST /v1/stt`, coercing SDK-level
+ * model options into xAI's wire format (booleans as `'true'`/`'false'`
+ * strings, numeric fields stringified, etc.).
+ *
+ * Wire-field mapping:
+ *   - `modelOptions.inverse_text_normalization` → `format` (xAI's chosen
+ *     wire-field name for the ITN boolean; the SDK surfaces it under the
+ *     clearer `inverse_text_normalization` key).
+ *   - `modelOptions.audio_format`, `sample_rate`, `multichannel`, `channels`,
+ *     `diarize` map to same-named form fields.
+ */
+export function buildTranscriptionFormData(options: {
+  file: File
+  language: string | undefined
+  modelOptions: GrokTranscriptionProviderOptions | undefined
+}): FormData {
+  const { file, language, modelOptions } = options
+  const form = new FormData()
+  form.set('file', file)
+  if (language) form.set('language', language)
+  if (modelOptions?.audio_format !== undefined) {
+    form.set('audio_format', modelOptions.audio_format)
+  }
+  if (modelOptions?.sample_rate !== undefined) {
+    form.set('sample_rate', String(modelOptions.sample_rate))
+  }
+  if (modelOptions?.inverse_text_normalization !== undefined) {
+    form.set(
+      'format',
+      modelOptions.inverse_text_normalization ? 'true' : 'false',
+    )
+  }
+  if (modelOptions?.multichannel !== undefined) {
+    form.set('multichannel', modelOptions.multichannel ? 'true' : 'false')
+  }
+  if (modelOptions?.channels !== undefined) {
+    form.set('channels', String(modelOptions.channels))
+  }
+  if (modelOptions?.diarize !== undefined) {
+    form.set('diarize', modelOptions.diarize ? 'true' : 'false')
+  }
+  return form
+}
+
+/**
+ * Creates a Grok transcription adapter with an explicit API key.
+ *
+ * @example
+ * ```typescript
+ * const adapter = createGrokTranscription('grok-stt', 'xai-...')
+ * const result = await generateTranscription({
+ *   adapter,
+ *   audio: audioFile,
+ *   language: 'en',
+ * })
+ * ```
+ */
+export function createGrokTranscription<TModel extends GrokTranscriptionModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<GrokTranscriptionConfig, 'apiKey'>,
+): GrokTranscriptionAdapter<TModel> {
+  return new GrokTranscriptionAdapter({ apiKey, ...config }, model)
+}
+
+/**
+ * Creates a Grok transcription adapter, reading the API key from
+ * `XAI_API_KEY` in the environment.
+ *
+ * @throws Error if `XAI_API_KEY` is not set.
+ */
+export function grokTranscription<TModel extends GrokTranscriptionModel>(
+  model: TModel,
+  config?: Omit<GrokTranscriptionConfig, 'apiKey'>,
+): GrokTranscriptionAdapter<TModel> {
+  const apiKey = getGrokApiKeyFromEnv()
+  return createGrokTranscription(model, apiKey, config)
+}

package/src/adapters/tts.ts

@@ -0,0 +1,260 @@
+import { BaseTTSAdapter } from '@tanstack/ai/adapters'
+import { arrayBufferToBase64, generateId, getGrokApiKeyFromEnv } from '../utils'
+import type { TTSOptions, TTSResult } from '@tanstack/ai'
+import type { GrokTTSModel } from '../model-meta'
+import type {
+  GrokTTSCodec,
+  GrokTTSProviderOptions,
+  GrokTTSVoice,
+} from '../audio/tts-provider-options'
+
+const DEFAULT_GROK_BASE_URL = 'https://api.x.ai/v1'
+
+/**
+ * Configuration for the Grok TTS adapter.
+ *
+ * Unlike chat/image/summarize adapters, TTS does not use the OpenAI SDK
+ * because xAI's `/v1/tts` endpoint is not OpenAI-compatible. This config
+ * is a minimal subset suitable for direct `fetch` calls.
+ */
+export interface GrokSpeechConfig {
+  apiKey: string
+  baseURL?: string
+  /** Additional headers to merge into every request (e.g., test IDs). */
+  defaultHeaders?: Record<string, string>
+}
+
+/**
+ * Grok Text-to-Speech Adapter.
+ *
+ * Talks to `POST {baseURL}/tts` per
+ * https://docs.x.ai/developers/model-capabilities/audio/text-to-speech
+ */
+export class GrokSpeechAdapter<
+  TModel extends GrokTTSModel,
+> extends BaseTTSAdapter<TModel, GrokTTSProviderOptions> {
+  readonly name = 'grok' as const
+
+  private readonly apiKey: string
+  private readonly baseURL: string
+  private readonly defaultHeaders: Record<string, string>
+
+  constructor(config: GrokSpeechConfig, model: TModel) {
+    super(model, config)
+    this.apiKey = config.apiKey
+    this.baseURL = (config.baseURL ?? DEFAULT_GROK_BASE_URL).replace(/\/+$/, '')
+    this.defaultHeaders = config.defaultHeaders ?? {}
+  }
+
+  async generateSpeech(
+    options: TTSOptions<GrokTTSProviderOptions>,
+  ): Promise<TTSResult> {
+    const { logger } = options
+    const { model, text, voice, format, modelOptions } = options
+
+    logger.request(`activity=generateSpeech provider=grok model=${model}`, {
+      provider: 'grok',
+      model,
+    })
+
+    const { body, codec, sampleRateForContentType } = buildTTSRequestBody({
+      text,
+      voice,
+      format,
+      modelOptions,
+    })
+
+    try {
+      const response = await fetch(`${this.baseURL}/tts`, {
+        method: 'POST',
+        headers: {
+          // `defaultHeaders` first so the adapter's Authorization / Content-Type
+          // always win — otherwise a caller-supplied `Authorization` header
+          // could silently clobber the bearer token.
+          ...this.defaultHeaders,
+          Authorization: `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(body),
+      })
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new Error(
+          `Grok TTS request failed: ${response.status} ${errorText}`,
+        )
+      }
+
+      const arrayBuffer = await response.arrayBuffer()
+      const audio = arrayBufferToBase64(arrayBuffer)
+
+      return {
+        id: generateId(this.name),
+        model,
+        audio,
+        format: codec,
+        contentType: getContentType(codec, sampleRateForContentType),
+      }
+    } catch (error) {
+      logger.errors('grok.generateSpeech fatal', {
+        error,
+        source: 'grok.generateSpeech',
+      })
+      throw error
+    }
+  }
+}
+
+/**
+ * Build the JSON body for `POST /v1/tts`, resolving codec / sample-rate / voice
+ * defaults in one place.
+ *
+ * Returns the request `body`, the resolved `codec`, and the `sampleRateForContentType`
+ * used by the caller to label the response via `getContentType`.
+ */
+export function buildTTSRequestBody(options: {
+  text: string
+  voice: string | undefined
+  format: TTSOptions['format'] | undefined
+  modelOptions: GrokTTSProviderOptions | undefined
+}): {
+  body: Record<string, unknown>
+  codec: GrokTTSCodec
+  sampleRateForContentType: number
+} {
+  const { text, voice, format, modelOptions } = options
+
+  const codec = pickCodec(modelOptions?.codec, format)
+
+  // Only forward `sample_rate` when either:
+  //   - the caller explicitly set `modelOptions.sample_rate`, or
+  //   - the codec's Content-Type carries the rate (pcm → audio/L16;rate=…).
+  // For mp3/wav/opus/aac/flac we leave sample_rate unset so xAI's server
+  // default applies.
+  const callerSampleRate = modelOptions?.sample_rate
+  // Default sample rate documented in GrokTTSProviderOptions is 24000 Hz —
+  // used only when we MUST attach a rate to the contentType (pcm) and the
+  // caller didn't pick one.
+  const pcmDefault = 24000
+  const needsRateInContentType = codec === 'pcm'
+
+  const outputFormat: Record<string, unknown> = { codec }
+  if (callerSampleRate !== undefined) {
+    outputFormat.sample_rate = callerSampleRate
+  } else if (needsRateInContentType) {
+    outputFormat.sample_rate = pcmDefault
+  }
+  if (codec === 'mp3' && modelOptions?.bit_rate !== undefined) {
+    outputFormat.bit_rate = modelOptions.bit_rate
+  }
+
+  // pcm embeds the rate in `audio/L16;rate=…`; mulaw/alaw embed it in
+  // `audio/PCMU;rate=…` / `audio/PCMA;rate=…` when non-default. mp3/wav
+  // don't carry a rate parameter so the value is unused for those.
+  const sampleRateForContentType = callerSampleRate ?? pcmDefault
+
+  const body: Record<string, unknown> = {
+    text,
+    voice_id: (voice as GrokTTSVoice | undefined) ?? 'eve',
+    language: modelOptions?.language ?? 'en',
+    output_format: outputFormat,
+  }
+  if (modelOptions?.optimize_streaming_latency !== undefined) {
+    body.optimize_streaming_latency = modelOptions.optimize_streaming_latency
+  }
+  if (modelOptions?.text_normalization !== undefined) {
+    body.text_normalization = modelOptions.text_normalization
+  }
+
+  return { body, codec, sampleRateForContentType }
+}
+
+/**
+ * Maps the cross-provider `TTSOptions.format` onto Grok's supported codecs.
+ * `opus`, `aac`, and `flac` are not supported by xAI TTS (which only exposes
+ * mp3/wav/pcm/mulaw/alaw) — we fall back to mp3. An explicit
+ * `modelOptions.codec` always wins.
+ */
+function pickCodec(
+  codecOverride: GrokTTSCodec | undefined,
+  format: TTSOptions['format'] | undefined,
+): GrokTTSCodec {
+  if (codecOverride) return codecOverride
+  if (!format) return 'mp3'
+  switch (format) {
+    case 'mp3':
+    case 'wav':
+    case 'pcm':
+      return format
+    case 'flac':
+    case 'opus':
+    case 'aac':
+      return 'mp3'
+    default:
+      return 'mp3'
+  }
+}
+
+export function getContentType(
+  codec: GrokTTSCodec,
+  sampleRate: number,
+): string {
+  switch (codec) {
+    case 'mp3':
+      return 'audio/mpeg'
+    case 'wav':
+      return 'audio/wav'
+    case 'pcm':
+      // `audio/L16` requires a `rate` parameter per RFC 3551/3555.
+      return `audio/L16;rate=${sampleRate}`
+    case 'mulaw':
+      // `audio/basic` is 8 kHz mono by RFC 2046 registration. For non-8kHz
+      // streams xAI still produces mulaw-encoded bytes at the requested
+      // rate, but the registered MIME can't carry that rate — so we use
+      // the non-standard but commonly-supported `audio/PCMU;rate=…` (RFC 3551
+      // RTP payload name) whenever the caller asked for a rate other than
+      // 8000, and keep `audio/basic` for the standard 8kHz case.
+      return sampleRate === 8000
+        ? 'audio/basic'
+        : `audio/PCMU;rate=${sampleRate}`
+    case 'alaw':
+      return sampleRate === 8000
+        ? 'audio/x-alaw-basic'
+        : `audio/PCMA;rate=${sampleRate}`
+  }
+}
+
+/**
+ * Creates a Grok speech (TTS) adapter with an explicit API key.
+ *
+ * @example
+ * ```typescript
+ * const adapter = createGrokSpeech('grok-tts', 'xai-...')
+ * const result = await generateSpeech({
+ *   adapter,
+ *   text: 'Hello from Grok',
+ *   voice: 'eve',
+ * })
+ * ```
+ */
+export function createGrokSpeech<TModel extends GrokTTSModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<GrokSpeechConfig, 'apiKey'>,
+): GrokSpeechAdapter<TModel> {
+  return new GrokSpeechAdapter({ apiKey, ...config }, model)
+}
+
+/**
+ * Creates a Grok speech (TTS) adapter, reading the API key from
+ * `XAI_API_KEY` in the environment.
+ *
+ * @throws Error if `XAI_API_KEY` is not set.
+ */
+export function grokSpeech<TModel extends GrokTTSModel>(
+  model: TModel,
+  config?: Omit<GrokSpeechConfig, 'apiKey'>,
+): GrokSpeechAdapter<TModel> {
+  const apiKey = getGrokApiKeyFromEnv()
+  return createGrokSpeech(model, apiKey, config)
+}

package/src/audio/transcription-provider-options.ts

@@ -0,0 +1,54 @@
+/**
+ * Grok STT supported audio formats.
+ * See https://docs.x.ai/developers/rest-api-reference/inference/voice
+ */
+export type GrokSTTAudioFormat =
+  | 'pcm'
+  | 'mulaw'
+  | 'alaw'
+  | 'wav'
+  | 'mp3'
+  | 'ogg'
+  | 'opus'
+  | 'flac'
+  | 'aac'
+  | 'mp4'
+  | 'm4a'
+  | 'mkv'
+
+/**
+ * Provider-specific options for Grok transcription (`POST /v1/stt`).
+ */
+export interface GrokTranscriptionProviderOptions {
+  /**
+   * The format of the provided audio. Required for raw codecs (pcm, mulaw, alaw).
+   */
+  audio_format?: GrokSTTAudioFormat
+  /**
+   * Sample rate of the audio (Hz). Required for raw codecs.
+   */
+  sample_rate?: number
+  /**
+   * Apply inverse text normalization (e.g. "one hundred" → "100"). Requires
+   * `language` to be set on the core `TranscriptionOptions`.
+   *
+   * NOTE: xAI's STT API exposes this on the wire as `format` (a boolean
+   * toggle). We surface it under the clearer name
+   * `inverse_text_normalization` on the SDK, and translate to the wire name
+   * inside the adapter.
+   */
+  inverse_text_normalization?: boolean
+  /**
+   * Treat the audio as multichannel. When enabled, `channels` must also be set.
+   */
+  multichannel?: boolean
+  /**
+   * Channel count for multichannel raw audio (2–8).
+   */
+  channels?: number
+  /**
+   * Enable speaker diarization. When true, response words include a `speaker`
+   * field.
+   */
+  diarize?: boolean
+}

package/src/audio/tts-provider-options.ts

@@ -0,0 +1,44 @@
+/**
+ * Grok TTS voice options.
+ * See https://docs.x.ai/developers/model-capabilities/audio/text-to-speech
+ */
+export type GrokTTSVoice = 'eve' | 'ara' | 'rex' | 'sal' | 'leo'
+
+/**
+ * Grok TTS output audio codecs.
+ * Grok does NOT support opus or aac; those formats are mapped to mp3.
+ */
+export type GrokTTSCodec = 'mp3' | 'wav' | 'pcm' | 'mulaw' | 'alaw'
+
+/**
+ * Provider-specific options for Grok TTS (`POST /v1/tts`).
+ */
+export interface GrokTTSProviderOptions {
+  /**
+   * BCP-47 language code (e.g., `en`, `zh`, `pt-BR`) or `'auto'` for detection.
+   * Defaults to `'en'` when not provided.
+   */
+  language?: string
+  /**
+   * Audio codec. Overrides the `format` field on `TTSOptions` when set.
+   */
+  codec?: GrokTTSCodec
+  /**
+   * Sample rate in Hz. Valid values: 8000, 16000, 22050, 24000, 44100, 48000.
+   * Defaults to 24000.
+   */
+  sample_rate?: 8000 | 16000 | 22050 | 24000 | 44100 | 48000
+  /**
+   * Bit rate for MP3 output. Ignored for other codecs.
+   * Valid values: 32000, 64000, 96000, 128000, 192000. Defaults to 128000.
+   */
+  bit_rate?: 32000 | 64000 | 96000 | 128000 | 192000
+  /**
+   * Set to 1 for lower latency streaming; 0 (default) for normal quality.
+   */
+  optimize_streaming_latency?: 0 | 1
+  /**
+   * Enable text normalization. Defaults to false.
+   */
+  text_normalization?: boolean
+}