@tanstack/ai-grok 0.6.7 → 0.7.0

package/src/realtime/index.ts

@@ -0,0 +1,18 @@
+// Token adapter for server-side use
+export { grokRealtimeToken } from './token'
+
+// Client adapter for browser use
+export { grokRealtime } from './adapter'
+
+// Types
+export type {
+  GrokRealtimeVoice,
+  GrokRealtimeTokenOptions,
+  GrokRealtimeOptions,
+  GrokTurnDetection,
+  GrokSemanticVADConfig,
+  GrokServerVADConfig,
+} from './types'
+
+// Re-export the realtime model type from the single source of truth.
+export type { GrokRealtimeModel } from '../model-meta'

package/src/realtime/realtime-contract.ts

@@ -0,0 +1,46 @@
+import type {
+  AnyClientTool,
+  AudioVisualization,
+  RealtimeEvent,
+  RealtimeEventHandler,
+  RealtimeSessionConfig,
+  RealtimeToken,
+} from '@tanstack/ai'
+
+/**
+ * Structural contract for the `RealtimeAdapter` / `RealtimeConnection` types
+ * from `@tanstack/ai-client`.
+ *
+ * We duplicate the shapes here (and verify structural compatibility in a
+ * dev-only type check — see `tests/realtime-contract.drift.test-d.ts`) so that
+ * `@tanstack/ai-grok` does not impose `@tanstack/ai-client` as a `peerDependency`.
+ * Consumers only need `@tanstack/ai-client` at the point where they actually
+ * construct a `RealtimeClient`, not when they import this adapter.
+ *
+ * If `@tanstack/ai-client` ever changes these interfaces, the drift check
+ * will fail and we must update this file in lockstep.
+ */
+
+export interface RealtimeAdapter {
+  provider: string
+  connect: (
+    token: RealtimeToken,
+    clientTools?: ReadonlyArray<AnyClientTool>,
+  ) => Promise<RealtimeConnection>
+}
+
+export interface RealtimeConnection {
+  disconnect: () => Promise<void>
+  startAudioCapture: () => Promise<void>
+  stopAudioCapture: () => void
+  sendText: (text: string) => void
+  sendImage: (imageData: string, mimeType: string) => void
+  sendToolResult: (callId: string, result: string) => void
+  updateSession: (config: Partial<RealtimeSessionConfig>) => void
+  interrupt: () => void
+  on: <TEvent extends RealtimeEvent>(
+    event: TEvent,
+    handler: RealtimeEventHandler<TEvent>,
+  ) => () => void
+  getAudioVisualization: () => AudioVisualization
+}

package/src/realtime/token.ts

@@ -0,0 +1,131 @@
+import { resolveDebugOption } from '@tanstack/ai/adapter-internals'
+import { getGrokApiKeyFromEnv } from '../utils'
+import type { RealtimeToken, RealtimeTokenAdapter } from '@tanstack/ai'
+import type { GrokRealtimeModel } from '../model-meta'
+import type {
+  GrokRealtimeSessionResponse,
+  GrokRealtimeTokenOptions,
+} from './types'
+
+const GROK_REALTIME_CLIENT_SECRETS_URL =
+  'https://api.x.ai/v1/realtime/client_secrets'
+
+const DEFAULT_TOKEN_FETCH_TIMEOUT_MS = 15_000
+
+/**
+ * Creates a Grok realtime token adapter.
+ *
+ * Generates ephemeral client secrets for browser-side WebRTC connections to
+ * the xAI Voice Agent API.
+ *
+ * @param options - Configuration options for the realtime session.
+ * @returns A RealtimeTokenAdapter for use with `realtimeToken()`.
+ *
+ * @example
+ * ```typescript
+ * import { realtimeToken } from '@tanstack/ai'
+ * import { grokRealtimeToken } from '@tanstack/ai-grok'
+ *
+ * const token = await realtimeToken({
+ *   adapter: grokRealtimeToken({ model: 'grok-voice-fast-1.0' }),
+ * })
+ * ```
+ */
+export function grokRealtimeToken(
+  options: GrokRealtimeTokenOptions = {},
+): RealtimeTokenAdapter {
+  const apiKey = getGrokApiKeyFromEnv()
+  const logger = resolveDebugOption(options.debug)
+
+  return {
+    provider: 'grok',
+
+    async generateToken(): Promise<RealtimeToken> {
+      const model: GrokRealtimeModel = options.model ?? 'grok-voice-fast-1.0'
+
+      logger.request(`activity=realtimeToken provider=grok model=${model}`, {
+        provider: 'grok',
+        model,
+      })
+
+      // xAI docs (docs.x.ai/developers/rest-api-reference/inference/voice)
+      // specify the body as `{ session: { model } }`. `expires_after` is
+      // available to shorten the default 600s TTL but we don't expose it
+      // yet — the caller can still call `generateToken()` more often if
+      // they want a shorter-lived session.
+      const requestBody: Record<string, unknown> = {
+        session: { model },
+      }
+
+      // Abort the fetch if xAI never responds. Without this the whole
+      // realtime connect flow hangs forever on a dead endpoint.
+      const controller = new AbortController()
+      const timeout = setTimeout(
+        () =>
+          controller.abort(new Error('Grok realtime token request timed out')),
+        DEFAULT_TOKEN_FETCH_TIMEOUT_MS,
+      )
+
+      try {
+        const response = await fetch(GROK_REALTIME_CLIENT_SECRETS_URL, {
+          method: 'POST',
+          headers: {
+            Authorization: `Bearer ${apiKey}`,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(requestBody),
+          signal: controller.signal,
+        })
+
+        if (!response.ok) {
+          const errorText = await response.text()
+          throw new Error(
+            `Grok realtime session creation failed: ${response.status} ${errorText}`,
+          )
+        }
+
+        const sessionData = (await response.json()) as
+          | Partial<GrokRealtimeSessionResponse>
+          | undefined
+
+        // Validate shape before dereferencing — xAI could return an error
+        // envelope with 200 status, or a partial response under protocol drift.
+        const clientSecret = sessionData?.client_secret
+        if (
+          !clientSecret ||
+          typeof clientSecret.value !== 'string' ||
+          typeof clientSecret.expires_at !== 'number' ||
+          !Number.isFinite(clientSecret.expires_at)
+        ) {
+          throw new Error(
+            'Grok realtime session response missing or malformed `client_secret`',
+          )
+        }
+        const sessionModel = sessionData.model ?? model
+
+        // xAI docs describe `expires_at` as a unix timestamp in seconds, but
+        // in practice different deployments have returned milliseconds. Treat
+        // any value that already looks like ms (>1e12 ≈ Sep 2001 in ms) as ms.
+        const raw = clientSecret.expires_at
+        const expiresAt = raw > 1e12 ? raw : raw * 1000
+
+        return {
+          provider: 'grok',
+          token: clientSecret.value,
+          expiresAt,
+          config: {
+            model: sessionModel,
+          },
+        }
+      } catch (error) {
+        logger.errors('grok.realtimeToken fatal', {
+          error,
+          source: 'grok.realtimeToken',
+        })
+        throw error
+      } finally {
+        clearTimeout(timeout)
+      }
+    },
+  }
+}

package/src/realtime/types.ts

@@ -0,0 +1,105 @@
+import type { DebugOption, VADConfig } from '@tanstack/ai'
+import type { GrokRealtimeModel } from '../model-meta'
+
+/**
+ * Grok realtime voice options (Voice Agent API).
+ * https://docs.x.ai/developers/model-capabilities/audio/voice-agent
+ */
+export type GrokRealtimeVoice = 'eve' | 'ara' | 'rex' | 'sal' | 'leo'
+
+/**
+ * Grok semantic VAD configuration.
+ */
+export interface GrokSemanticVADConfig {
+  type: 'semantic_vad'
+  /** Eagerness level for turn detection */
+  eagerness?: 'low' | 'medium' | 'high'
+}
+
+/**
+ * Grok server VAD configuration.
+ */
+export interface GrokServerVADConfig extends VADConfig {
+  type: 'server_vad'
+}
+
+/**
+ * Grok turn detection configuration.
+ */
+export type GrokTurnDetection =
+  | GrokSemanticVADConfig
+  | GrokServerVADConfig
+  | null
+
+/**
+ * Options for the Grok realtime token adapter.
+ */
+export interface GrokRealtimeTokenOptions {
+  /** Model to use (default: 'grok-voice-fast-1.0'). */
+  model?: GrokRealtimeModel
+  /**
+   * Enable debug logging for token creation.
+   *
+   * - `true`: log all categories via the default `ConsoleLogger`
+   * - `false`: silence everything including errors
+   * - `DebugConfig`: per-category toggles plus an optional custom `logger`
+   * - omitted: only the `errors` category is active (default behaviour)
+   */
+  debug?: DebugOption
+}
+
+/**
+ * Options for the Grok realtime client adapter.
+ */
+export interface GrokRealtimeOptions {
+  /** Connection mode (default: 'webrtc' in browser). */
+  connectionMode?: 'webrtc' | 'websocket'
+  /**
+   * Enable debug logging for this adapter.
+   *
+   * - `true`: log all categories via the default `ConsoleLogger`
+   * - `false`: silence everything including errors
+   * - `DebugConfig`: per-category toggles plus an optional custom `logger`
+   * - omitted: only the `errors` category is active (default behaviour)
+   */
+  debug?: DebugOption
+}
+
+/**
+ * Grok realtime session response from the `/v1/realtime/client_secrets`
+ * endpoint. Shape matches OpenAI's `/v1/realtime/sessions` response since
+ * xAI advertises its voice agent API as OpenAI-realtime-compatible.
+ */
+export interface GrokRealtimeSessionResponse {
+  id: string
+  object: string
+  model: string
+  modalities: Array<string>
+  instructions: string
+  voice: string
+  input_audio_format: string
+  output_audio_format: string
+  input_audio_transcription: {
+    model: string
+  } | null
+  turn_detection: {
+    type: string
+    threshold?: number
+    prefix_padding_ms?: number
+    silence_duration_ms?: number
+    eagerness?: string
+  } | null
+  tools: Array<{
+    type: string
+    name: string
+    description: string
+    parameters: Record<string, unknown>
+  }>
+  tool_choice: string
+  temperature: number
+  max_response_output_tokens: number | string
+  client_secret: {
+    value: string
+    expires_at: number
+  }
+}

package/src/utils/audio.ts

@@ -0,0 +1,217 @@
+/**
+ * Coerce the various audio input shapes accepted by `TranscriptionOptions.audio`
+ * into a `File` suitable for `multipart/form-data` uploads.
+ *
+ * For base64 string inputs we require an explicit MIME type — either via a
+ * `data:<mime>;base64,<payload>` URI prefix, or via the caller-provided
+ * `audioFormat` parameter. Bare base64 without either is rejected, because
+ * silently defaulting to `audio/mpeg` misreports non-mp3 audio to the server.
+ *
+ * The same rule applies to raw `ArrayBuffer` inputs: the caller must supply
+ * an `audioFormat` so we know what MIME type and extension to use.
+ */
+export function toAudioFile(
+  audio: string | File | Blob | ArrayBuffer,
+  audioFormat?: string,
+): File {
+  if (typeof File !== 'undefined' && audio instanceof File) {
+    // Prefer the caller-supplied `audioFormat` over a potentially empty or
+    // incorrect `File.type` — callers pass `audioFormat` precisely because
+    // they have more context than the browser does about the payload. If
+    // neither is set, fall through to the Blob-style error path below.
+    if (audioFormat) {
+      const mimeType = toMimeType(audioFormat)
+      return new File([audio], `audio.${extensionFor(mimeType)}`, {
+        type: mimeType,
+      })
+    }
+    if (audio.type) {
+      return audio
+    }
+    throw new Error(
+      'toAudioFile cannot infer type for File input with empty .type — pass an explicit audioFormat (e.g. "mp3", "wav", "audio/mpeg")',
+    )
+  }
+
+  if (typeof Blob !== 'undefined' && audio instanceof Blob) {
+    // Mirror the ArrayBuffer / bare-base64 paths: prefer the explicit
+    // audioFormat argument over the Blob's (often empty) .type. We refuse to
+    // fall back to `application/octet-stream` because that mislabels audio
+    // for the server.
+    const mimeType = audioFormat
+      ? toMimeType(audioFormat)
+      : audio.type || undefined
+    if (!mimeType) {
+      throw new Error(
+        'toAudioFile cannot infer type for Blob input with empty .type — pass an explicit audioFormat (e.g. "mp3", "wav", "audio/mpeg")',
+      )
+    }
+    return new File([audio], `audio.${extensionFor(mimeType)}`, {
+      type: mimeType,
+    })
+  }
+
+  if (audio instanceof ArrayBuffer) {
+    if (!audioFormat) {
+      throw new Error(
+        'toAudioFile cannot infer type for ArrayBuffer input — pass an explicit audioFormat (e.g. "mp3", "wav", "audio/mpeg")',
+      )
+    }
+    const mimeType = toMimeType(audioFormat)
+    return new File([audio], `audio.${extensionFor(mimeType)}`, {
+      type: mimeType,
+    })
+  }
+
+  if (typeof audio === 'string') {
+    if (audio.startsWith('data:')) {
+      const [header, base64Data] = audio.split(',')
+      // Fail loudly on malformed data: URIs instead of silently defaulting
+      // to `audio/mpeg` — the file's contract is that we never mislabel
+      // audio for the server.
+      const headerMatch = header?.match(/data:([^;]+)/)
+      const uriMimeType = headerMatch?.[1]
+      if (!uriMimeType) {
+        throw new Error(
+          'Malformed data: URI in toAudioFile: cannot parse MIME type — expected data:<mime>[;charset=…][;base64],<payload>',
+        )
+      }
+      if (base64Data === undefined || base64Data.trim() === '') {
+        throw new Error(
+          'Malformed data: URI in toAudioFile: missing base64 payload after comma',
+        )
+      }
+      // Caller-supplied `audioFormat` wins over the URI-embedded MIME: the
+      // caller has more context (the URI MIME may be wrong, or a generic
+      // `application/octet-stream`).
+      const mimeType = audioFormat ? toMimeType(audioFormat) : uriMimeType
+      const buffer = base64ToArrayBuffer(base64Data)
+      return new File([buffer], `audio.${extensionFor(mimeType)}`, {
+        type: mimeType,
+      })
+    }
+
+    if (!audioFormat) {
+      throw new Error(
+        'toAudioFile requires a data: URI (e.g. data:audio/wav;base64,...) or an explicit audioFormat argument — bare base64 strings have no MIME type to infer',
+      )
+    }
+
+    const buffer = base64ToArrayBuffer(audio)
+    const mimeType = toMimeType(audioFormat)
+    return new File([buffer], `audio.${extensionFor(mimeType)}`, {
+      type: mimeType,
+    })
+  }
+
+  throw new Error('Invalid audio input type')
+}
+
+function toMimeType(audioFormat: string): string {
+  // Accept either "audio/…" strings or bare extensions like "mp3".
+  if (audioFormat.includes('/')) return audioFormat
+  const ext = audioFormat.toLowerCase()
+  switch (ext) {
+    case 'mp3':
+      return 'audio/mpeg'
+    case 'wav':
+      return 'audio/wav'
+    case 'ogg':
+      return 'audio/ogg'
+    case 'opus':
+      return 'audio/opus'
+    case 'flac':
+      return 'audio/flac'
+    case 'aac':
+      return 'audio/aac'
+    case 'mp4':
+      return 'audio/mp4'
+    case 'm4a':
+      return 'audio/mp4'
+    case 'webm':
+      return 'audio/webm'
+    case 'pcm':
+      return 'audio/L16'
+    case 'mulaw':
+      return 'audio/basic'
+    case 'alaw':
+      return 'audio/x-alaw-basic'
+    default:
+      return `audio/${ext}`
+  }
+}
+
+function extensionFor(mimeType: string): string {
+  switch (mimeType) {
+    case 'audio/mpeg':
+      return 'mp3'
+    case 'audio/wav':
+    case 'audio/x-wav':
+      return 'wav'
+    case 'audio/ogg':
+      return 'ogg'
+    case 'audio/opus':
+      return 'opus'
+    case 'audio/flac':
+      return 'flac'
+    case 'audio/aac':
+      return 'aac'
+    case 'audio/mp4':
+      return 'm4a'
+    case 'audio/webm':
+      return 'webm'
+    case 'audio/L16':
+      return 'pcm'
+    case 'audio/basic':
+      return 'mulaw'
+    case 'audio/x-alaw-basic':
+      return 'alaw'
+    default: {
+      const slash = mimeType.indexOf('/')
+      if (slash === -1) return 'bin'
+      return mimeType.slice(slash + 1) || 'bin'
+    }
+  }
+}
+
+/**
+ * Cross-runtime ArrayBuffer → base64 conversion.
+ *
+ * Uses Node's `Buffer` when available (fastest path on server) and falls
+ * back to `btoa` + chunked `String.fromCharCode` everywhere else (browser,
+ * Cloudflare Workers, Bun, Deno). Chunking is required because a very
+ * large audio buffer spread into `String.fromCharCode(...bytes)` in one
+ * call can hit `Maximum call stack size exceeded`.
+ */
+export function arrayBufferToBase64(buffer: ArrayBuffer): string {
+  if (typeof Buffer !== 'undefined') {
+    return Buffer.from(buffer).toString('base64')
+  }
+  const bytes = new Uint8Array(buffer)
+  const chunkSize = 0x8000
+  let binary = ''
+  for (let i = 0; i < bytes.length; i += chunkSize) {
+    const end = Math.min(i + chunkSize, bytes.length)
+    binary += String.fromCharCode.apply(
+      null,
+      bytes.subarray(i, end) as unknown as Array<number>,
+    )
+  }
+  return btoa(binary)
+}
+
+function base64ToArrayBuffer(base64: string): ArrayBuffer {
+  let binary: string
+  try {
+    binary = atob(base64)
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err)
+    throw new Error(`Invalid base64 input to toAudioFile: ${msg}`)
+  }
+  const buffer = new ArrayBuffer(binary.length)
+  const bytes = new Uint8Array(buffer)
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i)
+  }
+  return buffer
+}

package/src/utils/index.ts

@@ -8,3 +8,4 @@ export {
   makeGrokStructuredOutputCompatible,
   transformNullsToUndefined,
 } from './schema-converter'
+export { toAudioFile, arrayBufferToBase64 } from './audio'