@tanstack/ai-gemini 0.19.1 → 0.20.0

package/src/realtime/types.ts

@@ -0,0 +1,94 @@
+import type {
+  ContextWindowCompressionConfig,
+  LiveConnectConstraints,
+  ThinkingConfig,
+} from '@google/genai'
+
+/**
+ * Gemini realtime voice options
+ */
+export type GeminiRealtimeVoice =
+  | 'Achernar'
+  | 'Achird'
+  | 'Algenib'
+  | 'Algieba'
+  | 'Alnilam'
+  | 'Aoede'
+  | 'Autonoe'
+  | 'Callirrhoe'
+  | 'Charon'
+  | 'Despina'
+  | 'Enceladus'
+  | 'Erinome'
+  | 'Fenrir'
+  | 'Gacrux'
+  | 'Iapetus'
+  | 'Kore'
+  | 'Laomedeia'
+  | 'Leda'
+  | 'Orus'
+  | 'Pulcherrima'
+  | 'Puck'
+  | 'Rasalgethi'
+  | 'Sadachbia'
+  | 'Sadaltager'
+  | 'Schedar'
+  | 'Sulafat'
+  | 'Umbriel'
+  | 'Vindemiatrix'
+  | 'Zephyr'
+  | 'Zubenelgenubi'
+
+/**
+ * Gemini realtime model options
+ */
+export type GeminiRealtimeModel = 'gemini-3.1-flash-live-preview'
+
+/**
+ * Options for the Gemini realtime client adapter
+ */
+export interface GeminiRealtimeOptions {
+  /** Connection mode (default: 'websocket' in browser) */
+  connectionMode?: 'websocket'
+  model?: GeminiRealtimeModel
+}
+
+export interface StrictLiveConnectionConstraints extends Omit<
+  LiveConnectConstraints,
+  'model'
+> {
+  model?: GeminiRealtimeModel
+}
+
+/**
+ * Options for the Gemini realtime token adapter
+ */
+export interface GeminiRealtimeTokenOptions {
+  expiresAt?: number
+  uses?: number
+  /**
+   * Config for LiveConnectConstraints for Auth Token creation.
+   *
+   * NOTE: Adding liveConnectConstraints will cause the API to ignore any config passed later to WebSocket.
+   */
+  liveConnectConstraints?: StrictLiveConnectionConstraints
+}
+
+/**
+ * Gemini-specific realtime options, passed through `providerOptions` on a
+ * realtime session config.
+ */
+export interface GeminiRealtimeProviderOptions {
+  /** Enable Google Search grounding (mutually exclusive with custom tools). */
+  googleGrounding?: boolean
+  /** Enable proactive audio so the model may choose when to respond. */
+  proactiveAudio?: boolean
+  /** Enable affective (emotion-aware) dialog. */
+  enableAffectiveDialog?: boolean
+  /** Context window compression configuration. */
+  contextWindowCompression?: ContextWindowCompressionConfig
+  /** Thinking configuration. */
+  thinkingConfig?: ThinkingConfig
+  /** BCP-47 language code for speech output. */
+  languageCode?: string
+}

package/src/realtime/utils.ts

@@ -0,0 +1,439 @@
+import type { GeminiLiveClient } from './client'
+
+/**
+ * Audio Worklet Processor for capturing and processing audio
+ */
+const captureWorkletCode = `
+class AudioCaptureProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.bufferSize = 512; // 32ms at 16kHz — per Gemini best practices (20-40ms chunks)
+    this.buffer = new Float32Array(this.bufferSize);
+    this.bufferIndex = 0;
+  }
+
+  process(inputs, outputs, parameters) {
+    const input = inputs[0];
+
+    if (input && input.length > 0) {
+      const inputChannel = input[0];
+
+      // Buffer the incoming audio
+      for (let i = 0; i < inputChannel.length; i++) {
+        this.buffer[this.bufferIndex++] = inputChannel[i];
+
+        // When buffer is full, send it to main thread
+        if (this.bufferIndex >= this.bufferSize) {
+          // Send the buffered audio to the main thread
+          this.port.postMessage({
+            type: "audio",
+            data: this.buffer.slice(),
+          });
+
+          // Reset buffer
+          this.bufferIndex = 0;
+        }
+      }
+    }
+
+    // Return true to keep the processor alive
+    return true;
+  }
+}
+
+// Register the processor
+registerProcessor("audio-capture-processor", AudioCaptureProcessor);`
+
+/**
+ * Audio Playback Worklet Processor for playing PCM audio.
+ * Uses an offset tracker instead of slice() to avoid allocations
+ * on the real-time audio thread.
+ */
+const playbackWorkletCode = `
+class PCMProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.audioQueue = [];
+    this.currentOffset = 0; // Track position in current buffer (avoids slice())
+
+    this.port.onmessage = (event) => {
+      if (event.data === "interrupt") {
+        // Clear the queue on interrupt
+        this.audioQueue = [];
+        this.currentOffset = 0;
+      } else if (event.data instanceof Float32Array) {
+        // Add audio data to the queue
+        this.audioQueue.push(event.data);
+      }
+    };
+  }
+
+  process(inputs, outputs, parameters) {
+    const output = outputs[0];
+    if (output.length === 0) return true;
+
+    const channel = output[0];
+    let outputIndex = 0;
+
+    // Fill the output buffer from the queue
+    while (outputIndex < channel.length && this.audioQueue.length > 0) {
+      const currentBuffer = this.audioQueue[0];
+
+      if (!currentBuffer || currentBuffer.length === 0) {
+        this.audioQueue.shift();
+        this.currentOffset = 0;
+        continue;
+      }
+
+      const remainingOutput = channel.length - outputIndex;
+      const remainingBuffer = currentBuffer.length - this.currentOffset;
+      const copyLength = Math.min(remainingOutput, remainingBuffer);
+
+      // Copy audio data to output using offset (no slice allocation)
+      for (let i = 0; i < copyLength; i++) {
+        channel[outputIndex++] = currentBuffer[this.currentOffset++];
+      }
+
+      // If we've consumed the entire buffer, move to the next one
+      if (this.currentOffset >= currentBuffer.length) {
+        this.audioQueue.shift();
+        this.currentOffset = 0;
+      }
+    }
+
+    // Fill remaining output with silence
+    while (outputIndex < channel.length) {
+      channel[outputIndex++] = 0;
+    }
+
+    return true;
+  }
+}
+
+registerProcessor("pcm-processor", PCMProcessor);`
+
+function calculateLevel(analyser: AnalyserNode): number {
+  const data = new Uint8Array(analyser.fftSize)
+  analyser.getByteTimeDomainData(data)
+
+  // Find peak deviation from center (128 is silence)
+  // This is more responsive than RMS for voice level meters
+  let maxDeviation = 0
+  for (const sample of data) {
+    const deviation = Math.abs(sample - 128)
+    if (deviation > maxDeviation) {
+      maxDeviation = deviation
+    }
+  }
+
+  // Normalize to 0-1 range (max deviation is 128)
+  // Scale by 1.5x so that ~66% amplitude reads as full scale
+  // This provides good visual feedback without pegging too early
+  const normalized = maxDeviation / 128
+  return Math.min(1, normalized * 1.5)
+}
+
+export function base64ToArrayBuffer(base64: string): ArrayBuffer {
+  const binary = atob(base64)
+  const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0))
+  return bytes.buffer
+}
+
+// Empty arrays for when visualization isn't available
+// frequencyBinCount = fftSize / 2 = 1024
+const emptyFrequencyData = new Uint8Array(1024)
+const emptyTimeDomainData = new Uint8Array(2048).fill(128) // 128 is silence
+
+export class AudioStreamer {
+  private audioContext: AudioContext | null = null
+  private audioWorklet: AudioWorkletNode | null = null
+  private mediaStream: MediaStream | null = null
+  private analyser: AnalyserNode | null = null
+  private isStreaming = false
+  private readonly sampleRate = 16000
+  private readonly client: GeminiLiveClient | null = null
+
+  constructor(client: GeminiLiveClient) {
+    this.client = client
+  }
+
+  get inputLevel() {
+    if (!this.analyser) return 0
+    return calculateLevel(this.analyser)
+  }
+
+  get inputFrequencyData() {
+    if (!this.analyser) return emptyFrequencyData
+    const data = new Uint8Array(this.analyser.frequencyBinCount)
+    this.analyser.getByteFrequencyData(data)
+    return data
+  }
+
+  get inputTimeDomainData() {
+    if (!this.analyser) return emptyTimeDomainData
+    const data = new Uint8Array(this.analyser.fftSize)
+    this.analyser.getByteTimeDomainData(data)
+    return data
+  }
+
+  get inputSampleRate() {
+    return this.sampleRate
+  }
+
+  async start() {
+    try {
+      const audioConstraints: MediaTrackConstraints = {
+        sampleRate: this.sampleRate,
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+      }
+
+      // Get microphone access
+      this.mediaStream = await navigator.mediaDevices.getUserMedia({
+        audio: audioConstraints,
+      })
+
+      // Check if native AGC is active
+      const track = this.mediaStream.getAudioTracks()[0]
+      const settings = track?.getSettings()
+
+      if (settings?.autoGainControl) {
+        console.warn('Native AGC not supported.')
+      }
+
+      // Create audio context
+      this.audioContext = new AudioContext({
+        sampleRate: this.sampleRate,
+      })
+
+      if (this.audioContext.state === 'suspended') {
+        await this.audioContext.resume()
+      }
+
+      const workletBlob = new Blob([captureWorkletCode], {
+        type: 'application/javascript',
+      })
+      const workletUrl = URL.createObjectURL(workletBlob)
+
+      // Load the audio worklet module, then release the blob URL.
+      await this.audioContext.audioWorklet.addModule(workletUrl)
+      URL.revokeObjectURL(workletUrl)
+
+      // Create the audio worklet node
+      this.audioWorklet = new AudioWorkletNode(
+        this.audioContext,
+        'audio-capture-processor',
+      )
+
+      // Set up message handling from the worklet
+      this.audioWorklet.port.onmessage = (event) => {
+        if (!this.isStreaming) return
+
+        if (event.data.type === 'audio') {
+          const inputData = event.data.data
+          const pcmData = this.convertToPCM16(inputData)
+          const base64Audio = this.arrayBufferToBase64(pcmData)
+
+          // Send to Gemini only if after setup complete
+          if (this.client?.isSetupComplete) {
+            this.client.sendAudioMessage(base64Audio)
+          }
+        }
+      }
+
+      // Create analyser for volume detection
+      this.analyser = this.audioContext.createAnalyser()
+      this.analyser.fftSize = 2048 // Larger size for more accurate level detection
+      this.analyser.smoothingTimeConstant = 0.3
+
+      // Connect the audio graph
+      const source = this.audioContext.createMediaStreamSource(this.mediaStream)
+      source.connect(this.analyser)
+      this.analyser.connect(this.audioWorklet)
+
+      // Start streaming
+      this.isStreaming = true
+    } catch (error) {
+      // Clean up the mic + audio context if setup failed partway through.
+      this.stop()
+      throw error
+    }
+  }
+
+  stop() {
+    this.isStreaming = false
+
+    if (this.audioWorklet) {
+      this.audioWorklet.disconnect()
+      this.audioWorklet.port.close()
+      this.audioWorklet = null
+    }
+
+    if (this.audioContext) {
+      void this.audioContext.close()
+      this.audioContext = null
+    }
+
+    if (this.mediaStream) {
+      this.mediaStream.getTracks().forEach((track) => track.stop())
+      this.mediaStream = null
+    }
+  }
+
+  startAudioCapture() {
+    if (this.mediaStream) {
+      for (const track of this.mediaStream.getAudioTracks()) {
+        track.enabled = true
+      }
+    }
+    this.isStreaming = true
+  }
+
+  stopAudioCapture() {
+    if (this.mediaStream) {
+      // Disable tracks rather than stopping them to allow re-enabling
+      for (const track of this.mediaStream.getAudioTracks()) {
+        track.enabled = false
+      }
+    }
+    this.isStreaming = false
+  }
+
+  private convertToPCM16(float32Array: Float32Array): ArrayBuffer {
+    const int16Array = new Int16Array(float32Array.length)
+    for (let i = 0; i < float32Array.length; i++) {
+      const sample = Math.max(-1, Math.min(1, float32Array[i] ?? 0))
+      int16Array[i] = sample * 0x7fff
+    }
+    return int16Array.buffer
+  }
+
+  private arrayBufferToBase64(buffer: ArrayBuffer): string {
+    const bytes = new Uint8Array(buffer)
+    const binary = String.fromCharCode(...bytes)
+    return btoa(binary)
+  }
+}
+
+export class AudioPlayer {
+  private audioContext: AudioContext | null = null
+  private workletNode: AudioWorkletNode | null = null
+  private gainNode: GainNode | null = null
+  private analyser: AnalyserNode | null = null
+  private isInitialized = false
+  private volume = 1.0
+  private readonly sampleRate = 24000
+
+  get outputLevel() {
+    if (!this.analyser) return 0
+    return calculateLevel(this.analyser)
+  }
+
+  get outputFrequencyData() {
+    if (!this.analyser) return emptyFrequencyData
+    const data = new Uint8Array(this.analyser.frequencyBinCount)
+    this.analyser.getByteFrequencyData(data)
+    return data
+  }
+
+  get outputTimeDomainData() {
+    if (!this.analyser) return emptyTimeDomainData
+    const data = new Uint8Array(this.analyser.fftSize)
+    this.analyser.getByteTimeDomainData(data)
+    return data
+  }
+
+  get outputSampleRate() {
+    return this.sampleRate
+  }
+
+  async init() {
+    if (this.isInitialized) return
+
+    try {
+      // Create audio context at 24kHz to match Gemini
+      this.audioContext = new AudioContext({
+        sampleRate: this.sampleRate,
+      })
+
+      const workletBlob = new Blob([playbackWorkletCode], {
+        type: 'application/javascript',
+      })
+      const workletUrl = URL.createObjectURL(workletBlob)
+
+      // Load the audio worklet module, then release the blob URL.
+      await this.audioContext.audioWorklet.addModule(workletUrl)
+      URL.revokeObjectURL(workletUrl)
+
+      // Create worklet node
+      this.workletNode = new AudioWorkletNode(
+        this.audioContext,
+        'pcm-processor',
+      )
+
+      // Create gain node for volume control
+      this.gainNode = this.audioContext.createGain()
+      this.gainNode.gain.value = this.volume
+
+      // Create analyser for volume detection
+      this.analyser = this.audioContext.createAnalyser()
+      this.analyser.fftSize = 2048 // Larger size for more accurate level detection
+      this.analyser.smoothingTimeConstant = 0.3
+
+      // Connect nodes
+      this.workletNode.connect(this.gainNode)
+      this.gainNode.connect(this.analyser)
+      this.analyser.connect(this.audioContext.destination)
+
+      this.isInitialized = true
+    } catch (error) {
+      // Release the audio context if initialization failed partway through.
+      this.destroy()
+      throw error
+    }
+  }
+
+  async play(pcmData: ArrayBuffer) {
+    if (!this.isInitialized) {
+      await this.init()
+    }
+
+    // Resume audio context if suspended
+    if (this.audioContext?.state === 'suspended') {
+      await this.audioContext.resume()
+    }
+
+    // Convert PCM16 LE to Float32
+    const inputArray = new Int16Array(pcmData)
+    const float32Data = new Float32Array(inputArray.length)
+    for (let i = 0; i < inputArray.length; i++) {
+      float32Data[i] = (inputArray[i] ?? 0) / 32768
+    }
+
+    // Send to worklet for playback
+    this.workletNode?.port.postMessage(float32Data)
+  }
+
+  /* Interrupt playback */
+  interrupt() {
+    if (this.workletNode) {
+      this.workletNode.port.postMessage('interrupt')
+    }
+  }
+
+  setVolume(volume: number) {
+    this.volume = Math.max(0, Math.min(1, volume))
+    if (this.gainNode) {
+      this.gainNode.gain.value = this.volume
+    }
+  }
+
+  destroy() {
+    if (this.audioContext) {
+      void this.audioContext.close()
+      this.audioContext = null
+    }
+    this.isInitialized = false
+  }
+}

package/src/video/video-provider-options.ts

@@ -1,25 +1,50 @@
 /**
- * Gemini Veo Video Generation Provider Options
+ * Gemini Video Generation Provider Options
  *
- * Based on https://ai.google.dev/gemini-api/docs/video
+ * Covers two request paths behind the one video adapter:
+ * - Veo models — long-running operations via `:predictLongRunning`
+ *   (https://ai.google.dev/gemini-api/docs/video)
+ * - Gemini Omni Flash — background jobs via the Interactions API
+ *   (https://ai.google.dev/gemini-api/docs/omni)
  *
  * @experimental Video generation is an experimental feature and may change.
  */
+import { GEMINI_INTERACTIONS_VIDEO_MODELS } from '../model-meta'
 import type { DurationOptions } from '@tanstack/ai/adapters'
-import type { GenerateVideosConfig } from '@google/genai'
+import type { GenerateVideosConfig, Interactions } from '@google/genai'
 import type { GEMINI_VIDEO_MODELS } from '../model-meta'
 
 /**
- * Model type for Gemini Veo video generation.
+ * Model type for Gemini video generation (Veo + Omni Flash).
  * @experimental Video generation is an experimental feature and may change.
  */
 export type GeminiVideoModel = (typeof GEMINI_VIDEO_MODELS)[number]
 
 /**
- * Supported aspect ratios for Veo video generation. This is the `size` value
- * for the Gemini video adapter — Veo expresses output shape as an aspect
- * ratio (plus an optional `resolution` in `modelOptions`), not pixel
- * dimensions.
+ * Video models served by the Interactions API (Gemini Omni Flash) rather
+ * than Veo's `:predictLongRunning` operations flow.
+ * @experimental Omni video generation is an experimental feature and may change.
+ */
+export type GeminiInteractionsVideoModel =
+  (typeof GEMINI_INTERACTIONS_VIDEO_MODELS)[number]
+
+/**
+ * Runtime guard for the Interactions-served video models.
+ * @experimental Omni video generation is an experimental feature and may change.
+ */
+export function isInteractionsVideoModel(
+  model: GeminiVideoModel,
+): model is GeminiInteractionsVideoModel {
+  return (GEMINI_INTERACTIONS_VIDEO_MODELS as ReadonlyArray<string>).includes(
+    model,
+  )
+}
+
+/**
+ * Supported aspect ratios for Gemini video generation. This is the `size`
+ * value for the Gemini video adapter — both Veo and Omni Flash express
+ * output shape as an aspect ratio (plus an optional `resolution` in Veo's
+ * `modelOptions`), not pixel dimensions.
  *
  * @experimental Video generation is an experimental feature and may change.
  */
@@ -49,13 +74,50 @@ export type GeminiVideoProviderOptions = Omit<
   | 'abortSignal'
 >
 
+/**
+ * Provider-specific options for Gemini Omni Flash video generation on the
+ * Interactions API.
+ *
+ * Derived from the SDK's `Interactions.CreateModelInteractionParamsNonStreaming`,
+ * minus the fields the adapter manages itself:
+ * - `model` / `input` — set from the adapter's model and the `prompt`
+ * - `stream` / `background` — the adapter always creates a background job
+ *   and polls it through the `generateVideo` jobs API
+ * - `response_modalities` / `response_format` — the adapter requests video
+ *   output and maps the top-level `size` option onto
+ *   `response_format.aspect_ratio`
+ * - `tools` / `response_mime_type` — not applicable to video generation
+ *
+ * Notable passthroughs:
+ * - `previous_interaction_id` — conversational video editing: chain a new
+ *   prompt onto a prior Omni interaction to refine its video
+ * - `generation_config.video_config.task` — pin the task mode
+ *   (`'text_to_video' | 'image_to_video' | 'reference_to_video' | 'edit'`)
+ *   instead of letting the model infer it
+ *
+ * @experimental Omni video generation is an experimental feature and may change.
+ */
+export type GeminiOmniVideoProviderOptions = Omit<
+  Interactions.CreateModelInteractionParamsNonStreaming,
+  | 'model'
+  | 'input'
+  | 'stream'
+  | 'background'
+  | 'response_modalities'
+  | 'response_format'
+  | 'response_mime_type'
+  | 'tools'
+>
+
 /**
  * Model-specific provider options mapping.
  *
  * @experimental Video generation is an experimental feature and may change.
  */
 export type GeminiVideoModelProviderOptionsByName = {
-  [TModel in GeminiVideoModel]: GeminiVideoProviderOptions
+  [TModel in GeminiVideoModel]: TModel extends GeminiInteractionsVideoModel
+    ? GeminiOmniVideoProviderOptions
+    : GeminiVideoProviderOptions
 }
 
 /**
@@ -70,17 +132,23 @@ export type GeminiVideoModelSizeByName = {
 /**
  * Per-model prompt input modalities. Every Veo model accepts image
  * conditioning inputs (first frame, last frame, reference images) alongside
- * the text prompt.
+ * the text prompt. Omni Flash additionally accepts video inputs (short
+ * reference clips / videos to edit).
  *
  * @experimental Video generation is an experimental feature and may change.
  */
 export type GeminiVideoModelInputModalitiesByName = {
-  [TModel in GeminiVideoModel]: readonly ['image']
+  [TModel in GeminiVideoModel]: TModel extends GeminiInteractionsVideoModel
+    ? readonly ['image', 'video']
+    : readonly ['image']
 }
 
 /**
- * Per-model duration unions (seconds, as numbers — the API's
- * `parameters.durationSeconds` field is numeric).
+ * Per-model duration unions (seconds, as numbers — Veo's
+ * `parameters.durationSeconds` field is numeric; Omni Flash accepts a
+ * continuous 3–10 second range, fractional seconds included, so it stays
+ * `number` — the adapter rejects out-of-range values at job creation,
+ * against the range entry below).
  *
  * @experimental Video generation is an experimental feature and may change.
  */
@@ -88,15 +156,21 @@ export type GeminiVideoModelDurationByName = {
   'veo-3.1-generate-preview': 4 | 6 | 8
   'veo-3.1-fast-generate-preview': 4 | 6 | 8
   'veo-3.1-lite-generate-preview': 4 | 6 | 8
+  'gemini-omni-flash-preview': number
 }
 
 /**
  * Runtime duration table backing `availableDurations()` / `snapDuration()`.
  *
- * Curated from the official Veo docs
+ * Veo values are curated from the official docs
  * (https://ai.google.dev/gemini-api/docs/video) — the Gemini OpenAPI spec
  * types the `:predictLongRunning` request's `parameters` as unconstrained,
  * so it carries no per-model duration information to derive these from.
+ * Omni Flash's 3–10s range was verified against the live API
+ * (2026-07-02): `response_format.duration` takes a `"<seconds>s"` string,
+ * fractional values are accepted, out-of-range values are rejected with
+ * "minimum allowed 3s" / "maximum allowed 10s", and omitting it defaults
+ * to a 10-second clip.
  *
  * @experimental Video generation is an experimental feature and may change.
  */
@@ -108,10 +182,16 @@ export const GEMINI_VIDEO_DURATIONS: {
   'veo-3.1-generate-preview': { kind: 'discrete', values: [4, 6, 8] },
   'veo-3.1-fast-generate-preview': { kind: 'discrete', values: [4, 6, 8] },
   'veo-3.1-lite-generate-preview': { kind: 'discrete', values: [4, 6, 8] },
+  'gemini-omni-flash-preview': {
+    kind: 'range',
+    min: 3,
+    max: 10,
+    unit: 'seconds',
+  },
 }
 
 /**
- * Look up the duration options for a Veo model.
+ * Look up the duration options for a Gemini video model.
  *
  * @experimental Video generation is an experimental feature and may change.
  */