@tanstack/ai-gemini 0.19.1 → 0.20.0

package/src/index.ts

@@ -61,9 +61,9 @@ export {
   type GeminiAudioProviderOptions,
 } from './adapters/audio'
 
-// Video / Veo generation adapter (experimental)
+// Video generation adapter — Veo + Gemini Omni Flash (experimental)
 /**
- * @experimental Veo video generation is an experimental feature and may change.
+ * @experimental Video generation is an experimental feature and may change.
  */
 export {
   GeminiVideoAdapter,
@@ -74,8 +74,11 @@ export {
 export {
   GEMINI_VIDEO_DURATIONS,
   getGeminiVideoDurationOptions,
+  isInteractionsVideoModel,
 } from './video/video-provider-options'
 export type {
+  GeminiInteractionsVideoModel,
+  GeminiOmniVideoProviderOptions,
   GeminiVideoModel,
   GeminiVideoModelDurationByName,
   GeminiVideoModelInputModalitiesByName,
@@ -96,6 +99,7 @@ export { GEMINI_TTS_MODELS as GeminiTTSModels } from './model-meta'
 export { GEMINI_TTS_VOICES as GeminiTTSVoices } from './model-meta'
 export { GEMINI_AUDIO_MODELS as GeminiAudioModels } from './model-meta'
 export { GEMINI_VIDEO_MODELS as GeminiVideoModels } from './model-meta'
+export { GEMINI_INTERACTIONS_VIDEO_MODELS as GeminiInteractionsVideoModels } from './model-meta'
 export type { GeminiModels as GeminiTextModel } from './model-meta'
 export type { GeminiImageModels as GeminiImageModel } from './model-meta'
 export type { GeminiTTSVoice } from './model-meta'
@@ -125,3 +129,17 @@ export type {
 
 // Export provider usage types
 export type { GeminiProviderUsageDetails } from './usage'
+
+// ============================================================================
+// Realtime (Voice) Adapters
+// ============================================================================
+
+export { geminiRealtime, geminiRealtimeToken } from './realtime/index'
+
+export type {
+  GeminiRealtimeModel,
+  GeminiRealtimeOptions,
+  GeminiRealtimeProviderOptions,
+  GeminiRealtimeTokenOptions,
+  GeminiRealtimeVoice,
+} from './realtime/index'

package/src/model-meta.ts

@@ -712,6 +712,37 @@ const VEO_3_1_LITE_PREVIEW = {
     GeminiCachedContentOptions
 >
 
+/**
+ * Gemini Omni Flash — multimodal video generation with conversational
+ * editing. Serves only the Interactions API (`generateContent` rejects it),
+ * so it routes through the interactions-based path of the video adapter,
+ * not Veo's `:predictLongRunning` flow. Pricing is per second of generated
+ * video ($0.10/sec). 720p / 24 FPS, 3–10 second clips (default 10s).
+ * @experimental Omni video generation is an experimental feature and may change.
+ */
+const GEMINI_OMNI_FLASH_PREVIEW = {
+  name: 'gemini-omni-flash-preview',
+  max_input_tokens: 1_048_576,
+  max_output_tokens: 1,
+  supports: {
+    input: ['text', 'image', 'video'],
+    output: ['video', 'audio'],
+  },
+  pricing: {
+    input: {
+      normal: 0,
+    },
+    output: {
+      normal: 0.1,
+    },
+  },
+} as const satisfies ModelMeta<
+  GeminiToolConfigOptions &
+    GeminiSafetyOptions &
+    GeminiCommonConfigOptions &
+    GeminiCachedContentOptions
+>
+
 const GEMINI_3_5_FLASH = {
   name: 'gemini-3.5-flash',
   max_input_tokens: 1_048_576,
@@ -845,13 +876,25 @@ export const GEMINI_TTS_VOICES = [
 export type GeminiTTSVoice = (typeof GEMINI_TTS_VOICES)[number]
 
 /**
- * Veo video generation models.
- * @experimental Veo video generation is an experimental feature and may change.
+ * Video generation models. Veo models run on the long-running
+ * `:predictLongRunning` flow; Gemini Omni Flash runs on the Interactions
+ * API — the video adapter routes by model.
+ * @experimental Video generation is an experimental feature and may change.
  */
 export const GEMINI_VIDEO_MODELS = [
   VEO_3_1_PREVIEW.name,
   VEO_3_1_FAST_PREVIEW.name,
   VEO_3_1_LITE_PREVIEW.name,
+  GEMINI_OMNI_FLASH_PREVIEW.name,
+] as const
+
+/**
+ * Video models served by the Interactions API rather than Veo's
+ * `:predictLongRunning` operations flow.
+ * @experimental Omni video generation is an experimental feature and may change.
+ */
+export const GEMINI_INTERACTIONS_VIDEO_MODELS = [
+  GEMINI_OMNI_FLASH_PREVIEW.name,
 ] as const
 
 // Manual type map for per-model provider options

package/src/realtime/adapter.ts

@@ -0,0 +1,311 @@
+import { createRealtimeEventEmitter } from '@tanstack/ai'
+import { AudioPlayer, AudioStreamer, base64ToArrayBuffer } from './utils'
+import { GeminiLiveClient } from './client'
+import type { LiveResponse } from './client'
+import type {
+  AnyClientTool,
+  AudioVisualization,
+  RealtimeAdapter,
+  RealtimeConnection,
+  RealtimeMessage,
+  RealtimeMode,
+  RealtimeToken,
+} from '@tanstack/ai'
+import type { GeminiRealtimeModel, GeminiRealtimeOptions } from './types'
+
+/**
+ * Creates a Gemini realtime adapter for client-side use.
+ *
+ * @param options - Optional configuration
+ * @returns A RealtimeAdapter for use with RealtimeClient
+ *
+ * @example
+ * ```typescript
+ * import { RealtimeClient } from '@tanstack/ai-client'
+ * import { geminiRealtime } from '@tanstack/ai-gemini'
+ *
+ * const client = new RealtimeClient({
+ *   getToken: () => fetch('/api/realtime-token').then(r => r.json()),
+ *   adapter: geminiRealtime(),
+ *   onGoAway: () => client.updateSession({ ... }) // Resume session with new config (available only for Gemini Live adapter)
+ * })
+ *
+ * ```
+ */
+export function geminiRealtime(
+  options: GeminiRealtimeOptions = {},
+): RealtimeAdapter {
+  return {
+    provider: 'gemini',
+
+    connect(
+      token: RealtimeToken,
+      clientTools?: ReadonlyArray<AnyClientTool>,
+    ): Promise<RealtimeConnection> {
+      return createWebSocketConnection(token, options.model, clientTools)
+    },
+  }
+}
+
+/**
+ * Creates a WebSocket connection to Gemini's realtime API
+ */
+async function createWebSocketConnection(
+  token: RealtimeToken,
+  model: GeminiRealtimeModel = 'gemini-3.1-flash-live-preview',
+  tools?: ReadonlyArray<AnyClientTool>,
+): Promise<RealtimeConnection> {
+  const { emit, on: realtimeEventEmitterOn } = createRealtimeEventEmitter()
+
+  // Current state
+  let currentMode: RealtimeMode = 'idle'
+  let currentMessageId: string | null = null
+  let messageIdCounter = 0
+
+  function generateMessageId(): string {
+    return `gemini-msg-${Date.now()}-${++messageIdCounter}`
+  }
+
+  const client = new GeminiLiveClient(token.token, model, tools)
+
+  let message: RealtimeMessage = {
+    id: '',
+    role: 'assistant',
+    timestamp: 0,
+    parts: [],
+  }
+
+  let pendingAssistantResponse = ''
+
+  client.onClose = () => {
+    emit('status_change', { status: 'idle' })
+    emit('mode_change', { mode: 'idle' })
+  }
+
+  client.onError = (error) => {
+    emit('error', { error })
+    emit('status_change', { status: 'error' })
+    emit('mode_change', { mode: 'idle' })
+  }
+
+  client.onReceiveResponse = (response: LiveResponse) => {
+    switch (response.type) {
+      case 'text':
+        message.parts.push({
+          type: 'text',
+          content: response.data,
+        })
+        break
+      case 'audio': {
+        // Decode once, then reuse for both the stored message part and playback.
+        const pcm = base64ToArrayBuffer(response.data.audioData)
+        message.parts.push({
+          type: 'audio',
+          transcript: response.data.transcript,
+          audioData: pcm,
+        })
+        if (currentMode !== 'speaking') {
+          currentMode = 'speaking'
+          emit('mode_change', { mode: 'speaking' })
+        }
+        audioPlayer.play(pcm).catch((error) => emit('error', { error }))
+        break
+      }
+      case 'go_away':
+        emit('go_away', { timeLeft: response.data.timeLeft })
+        break
+      case 'usage_metadata':
+        emit('usage', {
+          completionTokens: response.data.responseTokenCount ?? 0,
+          promptTokens: response.data.promptTokenCount ?? 0,
+          totalTokens: response.data.totalTokenCount ?? 0,
+        })
+        break
+      case 'input_transcription':
+        if (response.data.finished && currentMode !== 'thinking') {
+          currentMode = 'thinking'
+          emit('mode_change', { mode: 'thinking' })
+        }
+        emit('transcript', {
+          isFinal: response.data.finished,
+          transcript: response.data.text,
+          role: 'user',
+        })
+        break
+      case 'output_transcription':
+        pendingAssistantResponse += response.data.text
+        emit('transcript', {
+          isFinal: response.data.finished,
+          transcript: pendingAssistantResponse,
+          role: 'assistant',
+        })
+        break
+      case 'interrupted':
+        audioPlayer.interrupt()
+        currentMode = 'listening'
+        emit('mode_change', { mode: 'listening' })
+        emit('interrupted', { messageId: currentMessageId ?? undefined })
+        break
+      case 'tool_call':
+        for (const tool of response.data.functionCalls || []) {
+          if (tool.id && tool.name) {
+            emit('tool_call', {
+              toolCallId: tool.id,
+              input: tool.args,
+              toolName: tool.name,
+            })
+          }
+        }
+        break
+      case 'turn_complete':
+        currentMessageId = generateMessageId()
+        message.id = currentMessageId
+        message.timestamp = Date.now()
+        message.parts.push({
+          type: 'text',
+          content: pendingAssistantResponse,
+        })
+        emit('message_complete', { message })
+
+        pendingAssistantResponse = ''
+        message = {
+          id: '',
+          role: 'assistant',
+          timestamp: 0,
+          parts: [],
+        }
+        currentMode = 'listening'
+        emit('mode_change', { mode: 'listening' })
+        break
+      case 'setup_complete':
+        emit('status_change', { status: 'connected' })
+        break
+      case 'error':
+        emit('error', {
+          error: new Error(response.data),
+        })
+        break
+      case 'thought':
+      case 'session_resumption_update':
+        // Handled internally by GeminiLiveClient; not surfaced to the client.
+        break
+    }
+  }
+
+  await client.connect()
+
+  const audioStreamer = new AudioStreamer(client)
+  const audioPlayer = new AudioPlayer()
+  try {
+    await audioPlayer.init()
+    await audioStreamer.start()
+  } catch (error) {
+    // Tear down the socket + audio graph if mic/worklet setup fails (e.g. the
+    // user denied microphone access) rather than leaking an open connection.
+    audioStreamer.stop()
+    audioPlayer.destroy()
+    client.disconnect()
+    throw error
+  }
+
+  const connection: RealtimeConnection = {
+    async disconnect() {
+      audioStreamer.stop()
+      audioPlayer.destroy()
+      client.disconnect()
+      currentMode = 'idle'
+      emit('status_change', { status: 'idle' })
+    },
+
+    async startAudioCapture() {
+      // Audio capture is established during connection setup
+      audioStreamer.startAudioCapture()
+      currentMode = 'listening'
+      emit('mode_change', { mode: 'listening' })
+    },
+
+    stopAudioCapture() {
+      audioStreamer.stopAudioCapture()
+      currentMode = 'idle'
+      emit('mode_change', { mode: 'idle' })
+    },
+
+    sendText(text: string) {
+      client.sendTextMessage(text)
+      currentMode = 'thinking'
+      emit('mode_change', { mode: 'thinking' })
+    },
+
+    sendImage(imageData: string, mimeType: string) {
+      client.sendImageMessage(imageData, mimeType)
+      currentMode = 'thinking'
+      emit('mode_change', { mode: 'thinking' })
+    },
+
+    sendToolResult(callId: string, result: string) {
+      client.sendToolResponse([
+        {
+          id: callId,
+          response: {
+            result,
+          },
+        },
+      ])
+    },
+
+    updateSession(config) {
+      client.updateSession(config).catch((error) => emit('error', { error }))
+      emit('status_change', { status: 'reconnecting' })
+    },
+
+    updateToken(newToken) {
+      client.updateToken(newToken)
+      emit('status_change', { status: 'reconnecting' })
+    },
+
+    interrupt() {
+      audioPlayer.interrupt()
+      currentMode = 'listening'
+      emit('mode_change', { mode: 'listening' })
+      emit('interrupted', { messageId: currentMessageId ?? undefined })
+    },
+    on: realtimeEventEmitterOn,
+    getAudioVisualization(): AudioVisualization {
+      return {
+        get inputLevel() {
+          return audioStreamer.inputLevel
+        },
+
+        get outputLevel() {
+          return audioPlayer.outputLevel
+        },
+
+        getInputFrequencyData() {
+          return audioStreamer.inputFrequencyData
+        },
+
+        getOutputFrequencyData() {
+          return audioPlayer.outputFrequencyData
+        },
+
+        getInputTimeDomainData() {
+          return audioStreamer.inputTimeDomainData
+        },
+
+        getOutputTimeDomainData() {
+          return audioPlayer.outputTimeDomainData
+        },
+
+        get inputSampleRate() {
+          return audioStreamer.inputSampleRate
+        },
+
+        get outputSampleRate() {
+          return audioPlayer.outputSampleRate
+        },
+      }
+    },
+  }
+
+  return connection
+}