@tanstack/ai-gemini 0.9.0 → 0.10.0

package/src/adapters/tts.ts

@@ -4,21 +4,40 @@ import {
   generateId,
   getGeminiApiKeyFromEnv,
 } from '../utils'
+import { GEMINI_TTS_VOICES } from '../model-meta'
 import type { GEMINI_TTS_MODELS, GeminiTTSVoice } from '../model-meta'
 import type { TTSOptions, TTSResult } from '@tanstack/ai'
-import type { GoogleGenAI } from '@google/genai'
+import type { GoogleGenAI, SpeechConfig } from '@google/genai'
 import type { GeminiClientConfig } from '../utils'
 
+/**
+ * Configuration for a single speaker in a multi-speaker dialogue.
+ * Supported by Gemini 3.1 Flash TTS Preview and the 2.5 TTS models.
+ */
+export interface GeminiSpeakerVoiceConfig {
+  /** A name used in the prompt to refer to this speaker */
+  speaker: string
+  /** Voice configuration for this speaker */
+  voiceConfig: {
+    prebuiltVoiceConfig: {
+      voiceName: GeminiTTSVoice
+    }
+  }
+}
+
 /**
  * Provider-specific options for Gemini TTS
  *
  * @experimental Gemini TTS is an experimental feature.
  * @see https://ai.google.dev/gemini-api/docs/speech-generation
+ * @see https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview
  */
 export interface GeminiTTSProviderOptions {
   /**
-   * Voice configuration for TTS.
+   * Voice configuration for single-speaker TTS.
    * Choose from 30 available voices with different characteristics.
+   *
+   * Use `multiSpeakerVoiceConfig` instead for dialogues.
    */
   voiceConfig?: {
     prebuiltVoiceConfig?: {
@@ -30,11 +49,31 @@ export interface GeminiTTSProviderOptions {
     }
   }
 
+  /**
+   * Multi-speaker voice configuration (up to 2 speakers).
+   * Supported by Gemini 3.1 Flash TTS Preview and the 2.5 TTS models.
+   *
+   * Each speaker's lines in the prompt are prefixed with the name defined
+   * here, e.g.:
+   *
+   * ```text
+   * Joe: Hey, how's it going?
+   * Jane: Not bad, you?
+   * ```
+   */
+  multiSpeakerVoiceConfig?: {
+    speakerVoiceConfigs: Array<GeminiSpeakerVoiceConfig>
+  }
+
   /**
    * System instruction for controlling speech style.
    * Use natural language to describe the desired speaking style,
    * pace, tone, accent, or other characteristics.
    *
+   * With Gemini 3.1 Flash TTS, you can also use inline audio tags like
+   * `[whispering]`, `[laughs]`, `[excited]` directly in the input text
+   * to control delivery.
+   *
    * @example "Speak slowly and calmly, as if telling a bedtime story"
    * @example "Use an upbeat, enthusiastic tone with moderate pace"
    * @example "Speak with a British accent"
@@ -43,8 +82,8 @@ export interface GeminiTTSProviderOptions {
 
   /**
    * Language code hint for the speech synthesis.
-   * Gemini TTS supports 24 languages and can auto-detect,
-   * but you can provide a hint for better results.
+   * Gemini 3.1 Flash TTS supports 70+ languages with auto-detection;
+   * the 2.5 TTS models support 24 languages.
    *
    * @example "en-US" for American English
    * @example "es-ES" for Spanish (Spain)
@@ -85,7 +124,7 @@ export class GeminiTTSAdapter<
   private client: GoogleGenAI
 
   constructor(config: GeminiTTSConfig, model: TModel) {
-    super(config, model)
+    super(model, config)
     this.client = createGeminiClient(config)
   }
 
@@ -98,64 +137,236 @@ export class GeminiTTSAdapter<
   async generateSpeech(
     options: TTSOptions<GeminiTTSProviderOptions>,
   ): Promise<TTSResult> {
-    const { model, text, modelOptions } = options
+    const { model, text, modelOptions, voice, logger } = options
+
+    logger.request(`activity=generateSpeech provider=gemini model=${model}`, {
+      provider: 'gemini',
+      model,
+    })
+
+    const speechConfig: SpeechConfig = {}
 
-    const voiceConfig = modelOptions?.voiceConfig || {
-      prebuiltVoiceConfig: {
-        voiceName: 'Kore',
-      },
+    if (modelOptions?.multiSpeakerVoiceConfig) {
+      // Validate multi-speaker config: 1 or 2 speakers allowed.
+      const speakerConfigs =
+        modelOptions.multiSpeakerVoiceConfig.speakerVoiceConfigs
+      if (
+        !Array.isArray(speakerConfigs) ||
+        speakerConfigs.length < 1 ||
+        speakerConfigs.length > 2
+      ) {
+        throw new Error(
+          `Gemini TTS multiSpeakerVoiceConfig.speakerVoiceConfigs must contain 1 or 2 speakers; received ${Array.isArray(speakerConfigs) ? speakerConfigs.length : 'non-array'}.`,
+        )
+      }
+      speechConfig.multiSpeakerVoiceConfig =
+        modelOptions.multiSpeakerVoiceConfig
+    } else {
+      // Honor the standard TTSOptions.voice (used by every other TTS adapter)
+      // as a fallback for the prebuilt voice name. If an explicit
+      // modelOptions.voiceConfig is supplied its values win — but we still
+      // fall back to `voice` / 'Kore' if the supplied voiceConfig is missing
+      // prebuiltVoiceConfig.voiceName.
+      if (
+        voice !== undefined &&
+        !(GEMINI_TTS_VOICES as ReadonlyArray<string>).includes(voice)
+      ) {
+        throw new Error(
+          `Invalid Gemini TTS voice "${voice}". Valid voices are: ${GEMINI_TTS_VOICES.join(', ')}.`,
+        )
+      }
+      const defaultVoiceName = (voice as GeminiTTSVoice | undefined) ?? 'Kore'
+      const supplied = modelOptions?.voiceConfig
+      const resolvedVoiceName =
+        supplied?.prebuiltVoiceConfig?.voiceName ?? defaultVoiceName
+      speechConfig.voiceConfig = {
+        prebuiltVoiceConfig: { voiceName: resolvedVoiceName },
+      }
     }
 
-    const response = await this.client.models.generateContent({
-      model,
-      contents: [
-        {
-          role: 'user',
-          parts: [{ text }],
-        },
-      ],
-      config: {
-        responseModalities: ['AUDIO'],
-        speechConfig: {
-          voiceConfig,
-          ...(modelOptions?.languageCode && {
-            languageCode: modelOptions.languageCode,
+    if (modelOptions?.languageCode) {
+      speechConfig.languageCode = modelOptions.languageCode
+    }
+
+    try {
+      const response = await this.client.models.generateContent({
+        model,
+        contents: [
+          {
+            role: 'user',
+            parts: [{ text }],
+          },
+        ],
+        config: {
+          responseModalities: ['AUDIO'],
+          speechConfig,
+          // systemInstruction belongs inside `config` per the @google/genai
+          // contract — matches sibling Gemini adapters (summarize, text).
+          ...(modelOptions?.systemInstruction && {
+            systemInstruction: modelOptions.systemInstruction,
           }),
         },
-      },
-      ...(modelOptions?.systemInstruction && {
-        systemInstruction: modelOptions.systemInstruction,
-      }),
-    })
+      })
+
+      // Extract audio data from response
+      const candidate = response.candidates?.[0]
+      const parts = candidate?.content?.parts
 
-    // Extract audio data from response
-    const candidate = response.candidates?.[0]
-    const parts = candidate?.content?.parts
+      if (!parts || parts.length === 0) {
+        throw new Error('No audio output received from Gemini TTS')
+      }
 
-    if (!parts || parts.length === 0) {
-      throw new Error('No audio output received from Gemini TTS')
+      // Look for inline data (audio)
+      const audioPart = parts.find((part: any) =>
+        part.inlineData?.mimeType?.startsWith('audio/'),
+      )
+
+      if (!audioPart || !audioPart.inlineData || !audioPart.inlineData.data) {
+        throw new Error('No audio data in Gemini TTS response')
+      }
+
+      const audioBase64 = audioPart.inlineData.data
+      // mime is guaranteed by the `startsWith('audio/')` find predicate above.
+      const mimeType = audioPart.inlineData.mimeType as string
+
+      // Gemini TTS models return raw 16-bit LE PCM with a mime type like
+      // `audio/L16;codec=pcm;rate=24000`. That isn't playable in an <audio>
+      // element and the bare string isn't a usable file extension, so we
+      // prepend a RIFF/WAV header here and normalize the result to audio/wav.
+      const pcm = parsePcmMimeType(mimeType)
+      if (pcm) {
+        const wavBase64 = wrapPcmBase64AsWav(
+          audioBase64,
+          pcm.sampleRate,
+          pcm.channels,
+          pcm.bitsPerSample,
+        )
+        return {
+          id: generateId(this.name),
+          model,
+          audio: wavBase64,
+          format: 'wav',
+          contentType: 'audio/wav',
+        }
+      }
+
+      // Strip any mime parameters (e.g. `audio/ogg;codec=opus`) before pulling
+      // the subtype out as the file format.
+      const format = mimeType.split(';')[0]!.split('/')[1] || 'wav'
+
+      return {
+        id: generateId(this.name),
+        model,
+        audio: audioBase64,
+        format,
+        contentType: mimeType,
+      }
+    } catch (error) {
+      logger.errors('gemini.generateSpeech fatal', {
+        error,
+        source: 'gemini.generateSpeech',
+      })
+      throw error
     }
+  }
+}
+
+function parsePcmMimeType(
+  mimeType: string,
+): { sampleRate: number; channels: number; bitsPerSample: number } | undefined {
+  const normalized = mimeType.toLowerCase()
+  const subtype = normalized.split(';')[0]!.split('/')[1] ?? ''
+  // Exclude containerized wav (e.g. `audio/wav;codec=pcm`) — those already
+  // carry a RIFF header and must not be re-wrapped.
+  if (subtype.includes('wav')) return undefined
+
+  // Accept the variants Gemini and other providers actually emit:
+  //   - audio/L16;codec=pcm;rate=24000 (IANA PCM with bit depth in the type)
+  //   - audio/L24 and friends
+  //   - audio/pcm and audio/x-pcm
+  //   - anything else that explicitly tags codec=pcm and isn't wav-containered
+  const bitDepthMatch = /^audio\/l(\d+)/.exec(normalized)
+  const isPcm =
+    bitDepthMatch !== null ||
+    normalized.startsWith('audio/pcm') ||
+    normalized.startsWith('audio/x-pcm') ||
+    normalized.includes('codec=pcm')
+  if (!isPcm) return undefined
+
+  const rateMatch = /rate=(\d+)/.exec(normalized)
+  const channelsMatch = /channels=(\d+)/.exec(normalized)
+  // Default to 16-bit when the mime type doesn't specify — matches Gemini's
+  // audio/L16;codec=pcm;rate=24000 response.
+  const bitsPerSample = bitDepthMatch ? Number(bitDepthMatch[1]) : 16
+  return {
+    sampleRate: rateMatch ? Number(rateMatch[1]) : 24000,
+    channels: channelsMatch ? Number(channelsMatch[1]) : 1,
+    bitsPerSample,
+  }
+}
 
-    // Look for inline data (audio)
-    const audioPart = parts.find((part: any) =>
-      part.inlineData?.mimeType?.startsWith('audio/'),
+function wrapPcmBase64AsWav(
+  pcmBase64: string,
+  sampleRate: number,
+  channels = 1,
+  bitsPerSample = 16,
+): string {
+  // The WAV writer below emits a 16-bit PCM fmt chunk. If the source claims a
+  // different bit depth we'd be lying about the payload, so bail out loudly
+  // rather than producing a corrupt file.
+  if (bitsPerSample !== 16) {
+    throw new Error(
+      `Unsupported PCM bit depth ${bitsPerSample}: only 16-bit PCM can be wrapped as WAV.`,
     )
+  }
 
-    if (!audioPart || !audioPart.inlineData || !audioPart.inlineData.data) {
-      throw new Error('No audio data in Gemini TTS response')
-    }
+  const pcmBytes =
+    typeof Buffer !== 'undefined'
+      ? new Uint8Array(Buffer.from(pcmBase64, 'base64'))
+      : decodeBase64(pcmBase64)
 
-    const audioBase64 = audioPart.inlineData.data
-    const mimeType = audioPart.inlineData.mimeType || 'audio/wav'
-    const format = mimeType.split('/')[1] || 'wav'
+  const byteRate = (sampleRate * channels * bitsPerSample) / 8
+  const blockAlign = (channels * bitsPerSample) / 8
+  const dataSize = pcmBytes.byteLength
+  const buffer = new ArrayBuffer(44 + dataSize)
+  const view = new DataView(buffer)
 
-    return {
-      id: generateId(this.name),
-      model,
-      audio: audioBase64,
-      format,
-      contentType: mimeType,
-    }
+  writeAscii(view, 0, 'RIFF')
+  view.setUint32(4, 36 + dataSize, true)
+  writeAscii(view, 8, 'WAVE')
+  writeAscii(view, 12, 'fmt ')
+  view.setUint32(16, 16, true)
+  view.setUint16(20, 1, true)
+  view.setUint16(22, channels, true)
+  view.setUint32(24, sampleRate, true)
+  view.setUint32(28, byteRate, true)
+  view.setUint16(32, blockAlign, true)
+  view.setUint16(34, bitsPerSample, true)
+  writeAscii(view, 36, 'data')
+  view.setUint32(40, dataSize, true)
+  new Uint8Array(buffer, 44).set(pcmBytes)
+
+  if (typeof Buffer !== 'undefined') {
+    return Buffer.from(buffer).toString('base64')
+  }
+  let binary = ''
+  const bytes = new Uint8Array(buffer)
+  for (let i = 0; i < bytes.byteLength; i += 1) {
+    binary += String.fromCharCode(bytes[i]!)
+  }
+  return btoa(binary)
+}
+
+function decodeBase64(b64: string): Uint8Array {
+  const binary = atob(b64)
+  const out = new Uint8Array(binary.length)
+  for (let i = 0; i < binary.length; i += 1) out[i] = binary.charCodeAt(i)
+  return out
+}
+
+function writeAscii(view: DataView, offset: number, text: string): void {
+  for (let i = 0; i < text.length; i += 1) {
+    view.setUint8(offset + i, text.charCodeAt(i))
   }
 }
 
@@ -185,7 +396,9 @@ export function createGeminiSpeech<TModel extends GeminiTTSModel>(
   apiKey: string,
   config?: Omit<GeminiTTSConfig, 'apiKey'>,
 ): GeminiTTSAdapter<TModel> {
-  return new GeminiTTSAdapter({ apiKey, ...config }, model)
+  // Put apiKey LAST so caller-supplied config can't silently override the
+  // explicit argument.
+  return new GeminiTTSAdapter({ ...config, apiKey }, model)
 }
 
 /**

package/src/image/image-provider-options.ts

@@ -244,8 +244,28 @@ export function validateImageSize(
 }
 
 /**
- * Validates the number of images requested
- * Imagen models support 1-8 images per request (varies by model)
+ * Per-model caps on images per request.
+ * Imagen 3 and the Imagen 4 family all support up to 4 images per request
+ * via the Gemini API (the rumored 8-image tier is Vertex-only and isn't
+ * reachable through @google/genai today). Unknown models fall through to
+ * the shared cap defined below.
+ *
+ * @see https://ai.google.dev/gemini-api/docs/imagen
+ */
+const IMAGEN_MAX_IMAGES_BY_MODEL: Record<string, number> = {
+  'imagen-3.0-generate-002': 4,
+  'imagen-4.0-generate-001': 4,
+  'imagen-4.0-ultra-generate-001': 4,
+  'imagen-4.0-fast-generate-001': 4,
+}
+
+const DEFAULT_IMAGEN_MAX_IMAGES = 4
+
+/**
+ * Validates the number of images requested against the model's known cap.
+ * Uses a per-model table where available and falls back to the shared
+ * default otherwise — no more "some support up to 8" comments that don't
+ * match the error message.
  */
 export function validateNumberOfImages(
   model: string,
@@ -253,8 +273,8 @@ export function validateNumberOfImages(
 ): void {
   if (numberOfImages === undefined) return
 
-  // Most Imagen models support 1-4 images, some support up to 8
-  const maxImages = 4
+  const maxImages =
+    IMAGEN_MAX_IMAGES_BY_MODEL[model] ?? DEFAULT_IMAGEN_MAX_IMAGES
   if (numberOfImages < 1 || numberOfImages > maxImages) {
     throw new Error(
       `Invalid numberOfImages "${numberOfImages}" for model "${model}". ` +

package/src/index.ts

@@ -51,12 +51,26 @@ export {
   type GeminiTTSProviderOptions,
 } from './adapters/tts'
 
+// Audio / Lyria music generation adapter (experimental)
+/**
+ * @experimental Gemini Lyria music generation is an experimental feature and may change.
+ */
+export {
+  GeminiAudioAdapter,
+  createGeminiAudio,
+  geminiAudio,
+  type GeminiAudioConfig,
+  type GeminiAudioModel,
+  type GeminiAudioProviderOptions,
+} from './adapters/audio'
+
 // Re-export models from model-meta for convenience
 export { GEMINI_MODELS } from './model-meta'
 export { GEMINI_MODELS as GeminiTextModels } from './model-meta'
 export { GEMINI_IMAGE_MODELS as GeminiImageModels } from './model-meta'
 export { GEMINI_TTS_MODELS as GeminiTTSModels } from './model-meta'
 export { GEMINI_TTS_VOICES as GeminiTTSVoices } from './model-meta'
+export { GEMINI_AUDIO_MODELS as GeminiAudioModels } from './model-meta'
 export type { GeminiModels as GeminiTextModel } from './model-meta'
 export type { GeminiImageModels as GeminiImageModel } from './model-meta'
 export type { GeminiTTSVoice } from './model-meta'

package/src/model-meta.ts

@@ -295,7 +295,6 @@ const GEMINI_2_5_PRO_TTS = {
     input: ['text'],
     output: ['audio'],
     capabilities: ['audio_generation'],
-    tools: ['file_search'],
   },
   pricing: {
     input: {
@@ -455,7 +454,6 @@ const GEMINI_2_5_FLASH_TTS = {
     input: ['text'],
     output: ['audio'],
     capabilities: ['audio_generation', 'batch_api'],
-    tools: ['file_search'],
   },
   pricing: {
     input: {
@@ -472,6 +470,82 @@ const GEMINI_2_5_FLASH_TTS = {
     GeminiCachedContentOptions
 >
 
+/**
+ * Gemini 3.1 Flash TTS Preview - latest expressive TTS model with
+ * 200+ audio tags, 70+ languages, and multi-speaker dialogue support.
+ * @see https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview
+ */
+const GEMINI_3_1_FLASH_TTS = {
+  name: 'gemini-3.1-flash-tts-preview',
+  max_input_tokens: 32_768,
+  max_output_tokens: 16_384,
+  knowledge_cutoff: '2025-05-01',
+  supports: {
+    input: ['text'],
+    output: ['audio'],
+    capabilities: ['audio_generation', 'batch_api'],
+  },
+  pricing: {
+    input: {
+      normal: 0.5,
+    },
+    output: {
+      normal: 10,
+    },
+  },
+} as const satisfies ModelMeta<
+  GeminiToolConfigOptions &
+    GeminiSafetyOptions &
+    GeminiCommonConfigOptions &
+    GeminiCachedContentOptions
+>
+
+/**
+ * Lyria 3 Pro Preview — Google's flagship music generation model.
+ * Generates full-length songs with multiple verses, choruses, and bridges.
+ * Outputs MP3 or WAV at 48 kHz stereo.
+ * @see https://ai.google.dev/gemini-api/docs/models/lyria-3-pro-preview
+ */
+const LYRIA_3_PRO = {
+  name: 'lyria-3-pro-preview',
+  max_input_tokens: 131_072,
+  supports: {
+    input: ['text', 'image'],
+    output: ['audio'],
+    capabilities: ['audio_generation'],
+  },
+  pricing: {
+    input: {
+      normal: 0,
+    },
+    output: {
+      normal: 0,
+    },
+  },
+} as const satisfies ModelMeta
+
+/**
+ * Lyria 3 Clip Preview — 30-second music clips in MP3 format.
+ * @see https://ai.google.dev/gemini-api/docs/music-generation
+ */
+const LYRIA_3_CLIP = {
+  name: 'lyria-3-clip-preview',
+  max_input_tokens: 131_072,
+  supports: {
+    input: ['text', 'image'],
+    output: ['audio'],
+    capabilities: ['audio_generation'],
+  },
+  pricing: {
+    input: {
+      normal: 0,
+    },
+    output: {
+      normal: 0,
+    },
+  },
+} as const satisfies ModelMeta
+
 const GEMINI_2_5_FLASH_LITE = {
   name: 'gemini-2.5-flash-lite',
   max_input_tokens: 1_048_576,
@@ -580,7 +654,7 @@ const GEMINI_2_FLASH_IMAGE = {
   knowledge_cutoff: '2024-08-01',
   supports: {
     input: ['text', 'image', 'audio', 'video'],
-    output: ['text'],
+    output: ['text', 'image'],
     capabilities: ['batch_api', 'caching', 'structured_output'],
     tools: [],
   },
@@ -930,10 +1004,20 @@ export const GEMINI_IMAGE_MODELS = [
  * @experimental Gemini TTS is an experimental feature and may change.
  */
 export const GEMINI_TTS_MODELS = [
+  GEMINI_3_1_FLASH_TTS.name,
   GEMINI_2_5_FLASH_TTS.name,
   GEMINI_2_5_PRO_TTS.name,
 ] as const
 
+/**
+ * Audio generation models (Lyria music generation).
+ * @experimental Lyria music generation is an experimental feature and may change.
+ */
+export const GEMINI_AUDIO_MODELS = [
+  LYRIA_3_PRO.name,
+  LYRIA_3_CLIP.name,
+] as const
+
 /**
  * Available voice names for Gemini TTS
  * @see https://ai.google.dev/gemini-api/docs/speech-generation