@tanstack/ai-gemini 0.22.0 → 0.24.0

package/dist/esm/realtime/utils.js

@@ -219,7 +219,10 @@ var AudioStreamer = class {
 	}
 	convertToPCM16(float32Array) {
 		const int16Array = new Int16Array(float32Array.length);
-		for (let i = 0; i < float32Array.length; i++) int16Array[i] = Math.max(-1, Math.min(1, float32Array[i] ?? 0)) * 32767;
+		for (let i = 0; i < float32Array.length; i++) {
+			const sample = Math.max(-1, Math.min(1, float32Array[i] ?? 0));
+			int16Array[i] = sample * 32767;
+		}
 		return int16Array.buffer;
 	}
 	arrayBufferToBase64(buffer) {

package/dist/esm/realtime/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","names":[],"sources":["../../../src/realtime/utils.ts"],"sourcesContent":["import type { GeminiLiveClient } from './client'\n\n/**\n * Audio Worklet Processor for capturing and processing audio\n */\nconst captureWorkletCode = `\nclass AudioCaptureProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.bufferSize = 512; // 32ms at 16kHz — per Gemini best practices (20-40ms chunks)\n    this.buffer = new Float32Array(this.bufferSize);\n    this.bufferIndex = 0;\n  }\n\n  process(inputs, outputs, parameters) {\n    const input = inputs[0];\n\n    if (input && input.length > 0) {\n      const inputChannel = input[0];\n\n      // Buffer the incoming audio\n      for (let i = 0; i < inputChannel.length; i++) {\n        this.buffer[this.bufferIndex++] = inputChannel[i];\n\n        // When buffer is full, send it to main thread\n        if (this.bufferIndex >= this.bufferSize) {\n          // Send the buffered audio to the main thread\n          this.port.postMessage({\n            type: \"audio\",\n            data: this.buffer.slice(),\n          });\n\n          // Reset buffer\n          this.bufferIndex = 0;\n        }\n      }\n    }\n\n    // Return true to keep the processor alive\n    return true;\n  }\n}\n\n// Register the processor\nregisterProcessor(\"audio-capture-processor\", AudioCaptureProcessor);`\n\n/**\n * Audio Playback Worklet Processor for playing PCM audio.\n * Uses an offset tracker instead of slice() to avoid allocations\n * on the real-time audio thread.\n */\nconst playbackWorkletCode = `\nclass PCMProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.audioQueue = [];\n    this.currentOffset = 0; // Track position in current buffer (avoids slice())\n\n    this.port.onmessage = (event) => {\n      if (event.data === \"interrupt\") {\n        // Clear the queue on interrupt\n        this.audioQueue = [];\n        this.currentOffset = 0;\n      } else if (event.data instanceof Float32Array) {\n        // Add audio data to the queue\n        this.audioQueue.push(event.data);\n      }\n    };\n  }\n\n  process(inputs, outputs, parameters) {\n    const output = outputs[0];\n    if (output.length === 0) return true;\n\n    const channel = output[0];\n    let outputIndex = 0;\n\n    // Fill the output buffer from the queue\n    while (outputIndex < channel.length && this.audioQueue.length > 0) {\n      const currentBuffer = this.audioQueue[0];\n\n      if (!currentBuffer || currentBuffer.length === 0) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n        continue;\n      }\n\n      const remainingOutput = channel.length - outputIndex;\n      const remainingBuffer = currentBuffer.length - this.currentOffset;\n      const copyLength = Math.min(remainingOutput, remainingBuffer);\n\n      // Copy audio data to output using offset (no slice allocation)\n      for (let i = 0; i < copyLength; i++) {\n        channel[outputIndex++] = currentBuffer[this.currentOffset++];\n      }\n\n      // If we've consumed the entire buffer, move to the next one\n      if (this.currentOffset >= currentBuffer.length) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n      }\n    }\n\n    // Fill remaining output with silence\n    while (outputIndex < channel.length) {\n      channel[outputIndex++] = 0;\n    }\n\n    return true;\n  }\n}\n\nregisterProcessor(\"pcm-processor\", PCMProcessor);`\n\nfunction calculateLevel(analyser: AnalyserNode): number {\n  const data = new Uint8Array(analyser.fftSize)\n  analyser.getByteTimeDomainData(data)\n\n  // Find peak deviation from center (128 is silence)\n  // This is more responsive than RMS for voice level meters\n  let maxDeviation = 0\n  for (const sample of data) {\n    const deviation = Math.abs(sample - 128)\n    if (deviation > maxDeviation) {\n      maxDeviation = deviation\n    }\n  }\n\n  // Normalize to 0-1 range (max deviation is 128)\n  // Scale by 1.5x so that ~66% amplitude reads as full scale\n  // This provides good visual feedback without pegging too early\n  const normalized = maxDeviation / 128\n  return Math.min(1, normalized * 1.5)\n}\n\nexport function base64ToArrayBuffer(base64: string): ArrayBuffer {\n  const binary = atob(base64)\n  const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0))\n  return bytes.buffer\n}\n\n// Empty arrays for when visualization isn't available\n// frequencyBinCount = fftSize / 2 = 1024\nconst emptyFrequencyData = new Uint8Array(1024)\nconst emptyTimeDomainData = new Uint8Array(2048).fill(128) // 128 is silence\n\nexport class AudioStreamer {\n  private audioContext: AudioContext | null = null\n  private audioWorklet: AudioWorkletNode | null = null\n  private mediaStream: MediaStream | null = null\n  private analyser: AnalyserNode | null = null\n  private isStreaming = false\n  private readonly sampleRate = 16000\n  private readonly client: GeminiLiveClient | null = null\n\n  constructor(client: GeminiLiveClient) {\n    this.client = client\n  }\n\n  get inputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get inputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get inputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get inputSampleRate() {\n    return this.sampleRate\n  }\n\n  async start() {\n    try {\n      const audioConstraints: MediaTrackConstraints = {\n        sampleRate: this.sampleRate,\n        echoCancellation: true,\n        noiseSuppression: true,\n        autoGainControl: true,\n      }\n\n      // Get microphone access\n      this.mediaStream = await navigator.mediaDevices.getUserMedia({\n        audio: audioConstraints,\n      })\n\n      // Check if native AGC is active\n      const track = this.mediaStream.getAudioTracks()[0]\n      const settings = track?.getSettings()\n\n      if (settings?.autoGainControl) {\n        console.warn('Native AGC not supported.')\n      }\n\n      // Create audio context\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      if (this.audioContext.state === 'suspended') {\n        await this.audioContext.resume()\n      }\n\n      const workletBlob = new Blob([captureWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create the audio worklet node\n      this.audioWorklet = new AudioWorkletNode(\n        this.audioContext,\n        'audio-capture-processor',\n      )\n\n      // Set up message handling from the worklet\n      this.audioWorklet.port.onmessage = (event) => {\n        if (!this.isStreaming) return\n\n        if (event.data.type === 'audio') {\n          const inputData = event.data.data\n          const pcmData = this.convertToPCM16(inputData)\n          const base64Audio = this.arrayBufferToBase64(pcmData)\n\n          // Send to Gemini only if after setup complete\n          if (this.client?.isSetupComplete) {\n            this.client.sendAudioMessage(base64Audio)\n          }\n        }\n      }\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect the audio graph\n      const source = this.audioContext.createMediaStreamSource(this.mediaStream)\n      source.connect(this.analyser)\n      this.analyser.connect(this.audioWorklet)\n\n      // Start streaming\n      this.isStreaming = true\n    } catch (error) {\n      // Clean up the mic + audio context if setup failed partway through.\n      this.stop()\n      throw error\n    }\n  }\n\n  stop() {\n    this.isStreaming = false\n\n    if (this.audioWorklet) {\n      this.audioWorklet.disconnect()\n      this.audioWorklet.port.close()\n      this.audioWorklet = null\n    }\n\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n\n    if (this.mediaStream) {\n      this.mediaStream.getTracks().forEach((track) => track.stop())\n      this.mediaStream = null\n    }\n  }\n\n  startAudioCapture() {\n    if (this.mediaStream) {\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = true\n      }\n    }\n    this.isStreaming = true\n  }\n\n  stopAudioCapture() {\n    if (this.mediaStream) {\n      // Disable tracks rather than stopping them to allow re-enabling\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = false\n      }\n    }\n    this.isStreaming = false\n  }\n\n  private convertToPCM16(float32Array: Float32Array): ArrayBuffer {\n    const int16Array = new Int16Array(float32Array.length)\n    for (let i = 0; i < float32Array.length; i++) {\n      const sample = Math.max(-1, Math.min(1, float32Array[i] ?? 0))\n      int16Array[i] = sample * 0x7fff\n    }\n    return int16Array.buffer\n  }\n\n  private arrayBufferToBase64(buffer: ArrayBuffer): string {\n    const bytes = new Uint8Array(buffer)\n    const binary = String.fromCharCode(...bytes)\n    return btoa(binary)\n  }\n}\n\nexport class AudioPlayer {\n  private audioContext: AudioContext | null = null\n  private workletNode: AudioWorkletNode | null = null\n  private gainNode: GainNode | null = null\n  private analyser: AnalyserNode | null = null\n  private isInitialized = false\n  private volume = 1.0\n  private readonly sampleRate = 24000\n\n  get outputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get outputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get outputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get outputSampleRate() {\n    return this.sampleRate\n  }\n\n  async init() {\n    if (this.isInitialized) return\n\n    try {\n      // Create audio context at 24kHz to match Gemini\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      const workletBlob = new Blob([playbackWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create worklet node\n      this.workletNode = new AudioWorkletNode(\n        this.audioContext,\n        'pcm-processor',\n      )\n\n      // Create gain node for volume control\n      this.gainNode = this.audioContext.createGain()\n      this.gainNode.gain.value = this.volume\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect nodes\n      this.workletNode.connect(this.gainNode)\n      this.gainNode.connect(this.analyser)\n      this.analyser.connect(this.audioContext.destination)\n\n      this.isInitialized = true\n    } catch (error) {\n      // Release the audio context if initialization failed partway through.\n      this.destroy()\n      throw error\n    }\n  }\n\n  async play(pcmData: ArrayBuffer) {\n    if (!this.isInitialized) {\n      await this.init()\n    }\n\n    // Resume audio context if suspended\n    if (this.audioContext?.state === 'suspended') {\n      await this.audioContext.resume()\n    }\n\n    // Convert PCM16 LE to Float32\n    const inputArray = new Int16Array(pcmData)\n    const float32Data = new Float32Array(inputArray.length)\n    for (let i = 0; i < inputArray.length; i++) {\n      float32Data[i] = (inputArray[i] ?? 0) / 32768\n    }\n\n    // Send to worklet for playback\n    this.workletNode?.port.postMessage(float32Data)\n  }\n\n  /* Interrupt playback */\n  interrupt() {\n    if (this.workletNode) {\n      this.workletNode.port.postMessage('interrupt')\n    }\n  }\n\n  setVolume(volume: number) {\n    this.volume = Math.max(0, Math.min(1, volume))\n    if (this.gainNode) {\n      this.gainNode.gain.value = this.volume\n    }\n  }\n\n  destroy() {\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n    this.isInitialized = false\n  }\n}\n"],"mappings":";;;;AAKA,IAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8C3B,IAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+D5B,SAAS,eAAe,UAAgC;CACtD,MAAM,OAAO,IAAI,WAAW,SAAS,OAAO;CAC5C,SAAS,sBAAsB,IAAI;CAInC,IAAI,eAAe;CACnB,KAAK,MAAM,UAAU,MAAM;EACzB,MAAM,YAAY,KAAK,IAAI,SAAS,GAAG;EACvC,IAAI,YAAY,cACd,eAAe;CAEnB;CAKA,MAAM,aAAa,eAAe;CAClC,OAAO,KAAK,IAAI,GAAG,aAAa,GAAG;AACrC;AAEA,SAAgB,oBAAoB,QAA6B;CAC/D,MAAM,SAAS,KAAK,MAAM;CAE1B,OADc,WAAW,KAAK,SAAS,SAAS,KAAK,WAAW,CAAC,CAC1D,CAAA,CAAM;AACf;AAIA,IAAM,qCAAqB,IAAI,WAAW,IAAI;AAC9C,IAAM,uCAAsB,IAAI,WAAW,IAAI,EAAA,CAAE,KAAK,GAAG;AAEzD,IAAa,gBAAb,MAA2B;CACzB,eAA4C;CAC5C,eAAgD;CAChD,cAA0C;CAC1C,WAAwC;CACxC,cAAsB;CACtB,aAA8B;CAC9B,SAAmD;CAEnD,YAAY,QAA0B;EACpC,KAAK,SAAS;CAChB;CAEA,IAAI,aAAa;EACf,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,OAAO,eAAe,KAAK,QAAQ;CACrC;CAEA,IAAI,qBAAqB;EACvB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;EAC3D,KAAK,SAAS,qBAAqB,IAAI;EACvC,OAAO;CACT;CAEA,IAAI,sBAAsB;EACxB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;EACjD,KAAK,SAAS,sBAAsB,IAAI;EACxC,OAAO;CACT;CAEA,IAAI,kBAAkB;EACpB,OAAO,KAAK;CACd;CAEA,MAAM,QAAQ;EACZ,IAAI;GACF,MAAM,mBAA0C;IAC9C,YAAY,KAAK;IACjB,kBAAkB;IAClB,kBAAkB;IAClB,iBAAiB;GACnB;GAGA,KAAK,cAAc,MAAM,UAAU,aAAa,aAAa,EAC3D,OAAO,iBACT,CAAC;GAMD,KAHc,KAAK,YAAY,eAAe,CAAC,CAAC,EAC/B,EAAO,YAAY,EAAA,EAEtB,iBACZ,QAAQ,KAAK,2BAA2B;GAI1C,KAAK,eAAe,IAAI,aAAa,EACnC,YAAY,KAAK,WACnB,CAAC;GAED,IAAI,KAAK,aAAa,UAAU,aAC9B,MAAM,KAAK,aAAa,OAAO;GAGjC,MAAM,cAAc,IAAI,KAAK,CAAC,kBAAkB,GAAG,EACjD,MAAM,yBACR,CAAC;GACD,MAAM,aAAa,IAAI,gBAAgB,WAAW;GAGlD,MAAM,KAAK,aAAa,aAAa,UAAU,UAAU;GACzD,IAAI,gBAAgB,UAAU;GAG9B,KAAK,eAAe,IAAI,iBACtB,KAAK,cACL,yBACF;GAGA,KAAK,aAAa,KAAK,aAAa,UAAU;IAC5C,IAAI,CAAC,KAAK,aAAa;IAEvB,IAAI,MAAM,KAAK,SAAS,SAAS;KAC/B,MAAM,YAAY,MAAM,KAAK;KAC7B,MAAM,UAAU,KAAK,eAAe,SAAS;KAC7C,MAAM,cAAc,KAAK,oBAAoB,OAAO;KAGpD,IAAI,KAAK,QAAQ,iBACf,KAAK,OAAO,iBAAiB,WAAW;IAE5C;GACF;GAGA,KAAK,WAAW,KAAK,aAAa,eAAe;GACjD,KAAK,SAAS,UAAU;GACxB,KAAK,SAAS,wBAAwB;GAItC,KADoB,aAAa,wBAAwB,KAAK,WAC9D,CAAA,CAAO,QAAQ,KAAK,QAAQ;GAC5B,KAAK,SAAS,QAAQ,KAAK,YAAY;GAGvC,KAAK,cAAc;EACrB,SAAS,OAAO;GAEd,KAAK,KAAK;GACV,MAAM;EACR;CACF;CAEA,OAAO;EACL,KAAK,cAAc;EAEnB,IAAI,KAAK,cAAc;GACrB,KAAK,aAAa,WAAW;GAC7B,KAAK,aAAa,KAAK,MAAM;GAC7B,KAAK,eAAe;EACtB;EAEA,IAAI,KAAK,cAAc;GACrB,KAAU,aAAa,MAAM;GAC7B,KAAK,eAAe;EACtB;EAEA,IAAI,KAAK,aAAa;GACpB,KAAK,YAAY,UAAU,CAAC,CAAC,SAAS,UAAU,MAAM,KAAK,CAAC;GAC5D,KAAK,cAAc;EACrB;CACF;CAEA,oBAAoB;EAClB,IAAI,KAAK,aACP,KAAK,MAAM,SAAS,KAAK,YAAY,eAAe,GAClD,MAAM,UAAU;EAGpB,KAAK,cAAc;CACrB;CAEA,mBAAmB;EACjB,IAAI,KAAK,aAEP,KAAK,MAAM,SAAS,KAAK,YAAY,eAAe,GAClD,MAAM,UAAU;EAGpB,KAAK,cAAc;CACrB;CAEA,eAAuB,cAAyC;EAC9D,MAAM,aAAa,IAAI,WAAW,aAAa,MAAM;EACrD,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,QAAQ,KAEvC,WAAW,KADI,KAAK,IAAI,IAAI,KAAK,IAAI,GAAG,aAAa,MAAM,CAAC,CAC5C,IAAS;EAE3B,OAAO,WAAW;CACpB;CAEA,oBAA4B,QAA6B;EACvD,MAAM,QAAQ,IAAI,WAAW,MAAM;EACnC,MAAM,SAAS,OAAO,aAAa,GAAG,KAAK;EAC3C,OAAO,KAAK,MAAM;CACpB;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB,eAA4C;CAC5C,cAA+C;CAC/C,WAAoC;CACpC,WAAwC;CACxC,gBAAwB;CACxB,SAAiB;CACjB,aAA8B;CAE9B,IAAI,cAAc;EAChB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,OAAO,eAAe,KAAK,QAAQ;CACrC;CAEA,IAAI,sBAAsB;EACxB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;EAC3D,KAAK,SAAS,qBAAqB,IAAI;EACvC,OAAO;CACT;CAEA,IAAI,uBAAuB;EACzB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;EACjD,KAAK,SAAS,sBAAsB,IAAI;EACxC,OAAO;CACT;CAEA,IAAI,mBAAmB;EACrB,OAAO,KAAK;CACd;CAEA,MAAM,OAAO;EACX,IAAI,KAAK,eAAe;EAExB,IAAI;GAEF,KAAK,eAAe,IAAI,aAAa,EACnC,YAAY,KAAK,WACnB,CAAC;GAED,MAAM,cAAc,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAClD,MAAM,yBACR,CAAC;GACD,MAAM,aAAa,IAAI,gBAAgB,WAAW;GAGlD,MAAM,KAAK,aAAa,aAAa,UAAU,UAAU;GACzD,IAAI,gBAAgB,UAAU;GAG9B,KAAK,cAAc,IAAI,iBACrB,KAAK,cACL,eACF;GAGA,KAAK,WAAW,KAAK,aAAa,WAAW;GAC7C,KAAK,SAAS,KAAK,QAAQ,KAAK;GAGhC,KAAK,WAAW,KAAK,aAAa,eAAe;GACjD,KAAK,SAAS,UAAU;GACxB,KAAK,SAAS,wBAAwB;GAGtC,KAAK,YAAY,QAAQ,KAAK,QAAQ;GACtC,KAAK,SAAS,QAAQ,KAAK,QAAQ;GACnC,KAAK,SAAS,QAAQ,KAAK,aAAa,WAAW;GAEnD,KAAK,gBAAgB;EACvB,SAAS,OAAO;GAEd,KAAK,QAAQ;GACb,MAAM;EACR;CACF;CAEA,MAAM,KAAK,SAAsB;EAC/B,IAAI,CAAC,KAAK,eACR,MAAM,KAAK,KAAK;EAIlB,IAAI,KAAK,cAAc,UAAU,aAC/B,MAAM,KAAK,aAAa,OAAO;EAIjC,MAAM,aAAa,IAAI,WAAW,OAAO;EACzC,MAAM,cAAc,IAAI,aAAa,WAAW,MAAM;EACtD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KACrC,YAAY,MAAM,WAAW,MAAM,KAAK;EAI1C,KAAK,aAAa,KAAK,YAAY,WAAW;CAChD;CAGA,YAAY;EACV,IAAI,KAAK,aACP,KAAK,YAAY,KAAK,YAAY,WAAW;CAEjD;CAEA,UAAU,QAAgB;EACxB,KAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;EAC7C,IAAI,KAAK,UACP,KAAK,SAAS,KAAK,QAAQ,KAAK;CAEpC;CAEA,UAAU;EACR,IAAI,KAAK,cAAc;GACrB,KAAU,aAAa,MAAM;GAC7B,KAAK,eAAe;EACtB;EACA,KAAK,gBAAgB;CACvB;AACF"}
+{"version":3,"file":"utils.js","names":[],"sources":["../../../src/realtime/utils.ts"],"sourcesContent":["import type { GeminiLiveClient } from './client'\n\n/**\n * Audio Worklet Processor for capturing and processing audio\n */\nconst captureWorkletCode = `\nclass AudioCaptureProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.bufferSize = 512; // 32ms at 16kHz — per Gemini best practices (20-40ms chunks)\n    this.buffer = new Float32Array(this.bufferSize);\n    this.bufferIndex = 0;\n  }\n\n  process(inputs, outputs, parameters) {\n    const input = inputs[0];\n\n    if (input && input.length > 0) {\n      const inputChannel = input[0];\n\n      // Buffer the incoming audio\n      for (let i = 0; i < inputChannel.length; i++) {\n        this.buffer[this.bufferIndex++] = inputChannel[i];\n\n        // When buffer is full, send it to main thread\n        if (this.bufferIndex >= this.bufferSize) {\n          // Send the buffered audio to the main thread\n          this.port.postMessage({\n            type: \"audio\",\n            data: this.buffer.slice(),\n          });\n\n          // Reset buffer\n          this.bufferIndex = 0;\n        }\n      }\n    }\n\n    // Return true to keep the processor alive\n    return true;\n  }\n}\n\n// Register the processor\nregisterProcessor(\"audio-capture-processor\", AudioCaptureProcessor);`\n\n/**\n * Audio Playback Worklet Processor for playing PCM audio.\n * Uses an offset tracker instead of slice() to avoid allocations\n * on the real-time audio thread.\n */\nconst playbackWorkletCode = `\nclass PCMProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.audioQueue = [];\n    this.currentOffset = 0; // Track position in current buffer (avoids slice())\n\n    this.port.onmessage = (event) => {\n      if (event.data === \"interrupt\") {\n        // Clear the queue on interrupt\n        this.audioQueue = [];\n        this.currentOffset = 0;\n      } else if (event.data instanceof Float32Array) {\n        // Add audio data to the queue\n        this.audioQueue.push(event.data);\n      }\n    };\n  }\n\n  process(inputs, outputs, parameters) {\n    const output = outputs[0];\n    if (output.length === 0) return true;\n\n    const channel = output[0];\n    let outputIndex = 0;\n\n    // Fill the output buffer from the queue\n    while (outputIndex < channel.length && this.audioQueue.length > 0) {\n      const currentBuffer = this.audioQueue[0];\n\n      if (!currentBuffer || currentBuffer.length === 0) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n        continue;\n      }\n\n      const remainingOutput = channel.length - outputIndex;\n      const remainingBuffer = currentBuffer.length - this.currentOffset;\n      const copyLength = Math.min(remainingOutput, remainingBuffer);\n\n      // Copy audio data to output using offset (no slice allocation)\n      for (let i = 0; i < copyLength; i++) {\n        channel[outputIndex++] = currentBuffer[this.currentOffset++];\n      }\n\n      // If we've consumed the entire buffer, move to the next one\n      if (this.currentOffset >= currentBuffer.length) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n      }\n    }\n\n    // Fill remaining output with silence\n    while (outputIndex < channel.length) {\n      channel[outputIndex++] = 0;\n    }\n\n    return true;\n  }\n}\n\nregisterProcessor(\"pcm-processor\", PCMProcessor);`\n\nfunction calculateLevel(analyser: AnalyserNode): number {\n  const data = new Uint8Array(analyser.fftSize)\n  analyser.getByteTimeDomainData(data)\n\n  // Find peak deviation from center (128 is silence)\n  // This is more responsive than RMS for voice level meters\n  let maxDeviation = 0\n  for (const sample of data) {\n    const deviation = Math.abs(sample - 128)\n    if (deviation > maxDeviation) {\n      maxDeviation = deviation\n    }\n  }\n\n  // Normalize to 0-1 range (max deviation is 128)\n  // Scale by 1.5x so that ~66% amplitude reads as full scale\n  // This provides good visual feedback without pegging too early\n  const normalized = maxDeviation / 128\n  return Math.min(1, normalized * 1.5)\n}\n\nexport function base64ToArrayBuffer(base64: string): ArrayBuffer {\n  const binary = atob(base64)\n  const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0))\n  return bytes.buffer\n}\n\n// Empty arrays for when visualization isn't available\n// frequencyBinCount = fftSize / 2 = 1024\nconst emptyFrequencyData = new Uint8Array(1024)\nconst emptyTimeDomainData = new Uint8Array(2048).fill(128) // 128 is silence\n\nexport class AudioStreamer {\n  private audioContext: AudioContext | null = null\n  private audioWorklet: AudioWorkletNode | null = null\n  private mediaStream: MediaStream | null = null\n  private analyser: AnalyserNode | null = null\n  private isStreaming = false\n  private readonly sampleRate = 16000\n  private readonly client: GeminiLiveClient | null = null\n\n  constructor(client: GeminiLiveClient) {\n    this.client = client\n  }\n\n  get inputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get inputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get inputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get inputSampleRate() {\n    return this.sampleRate\n  }\n\n  async start() {\n    try {\n      const audioConstraints: MediaTrackConstraints = {\n        sampleRate: this.sampleRate,\n        echoCancellation: true,\n        noiseSuppression: true,\n        autoGainControl: true,\n      }\n\n      // Get microphone access\n      this.mediaStream = await navigator.mediaDevices.getUserMedia({\n        audio: audioConstraints,\n      })\n\n      // Check if native AGC is active\n      const track = this.mediaStream.getAudioTracks()[0]\n      const settings = track?.getSettings()\n\n      if (settings?.autoGainControl) {\n        console.warn('Native AGC not supported.')\n      }\n\n      // Create audio context\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      if (this.audioContext.state === 'suspended') {\n        await this.audioContext.resume()\n      }\n\n      const workletBlob = new Blob([captureWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create the audio worklet node\n      this.audioWorklet = new AudioWorkletNode(\n        this.audioContext,\n        'audio-capture-processor',\n      )\n\n      // Set up message handling from the worklet\n      this.audioWorklet.port.onmessage = (event) => {\n        if (!this.isStreaming) return\n\n        if (event.data.type === 'audio') {\n          const inputData = event.data.data\n          const pcmData = this.convertToPCM16(inputData)\n          const base64Audio = this.arrayBufferToBase64(pcmData)\n\n          // Send to Gemini only if after setup complete\n          if (this.client?.isSetupComplete) {\n            this.client.sendAudioMessage(base64Audio)\n          }\n        }\n      }\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect the audio graph\n      const source = this.audioContext.createMediaStreamSource(this.mediaStream)\n      source.connect(this.analyser)\n      this.analyser.connect(this.audioWorklet)\n\n      // Start streaming\n      this.isStreaming = true\n    } catch (error) {\n      // Clean up the mic + audio context if setup failed partway through.\n      this.stop()\n      throw error\n    }\n  }\n\n  stop() {\n    this.isStreaming = false\n\n    if (this.audioWorklet) {\n      this.audioWorklet.disconnect()\n      this.audioWorklet.port.close()\n      this.audioWorklet = null\n    }\n\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n\n    if (this.mediaStream) {\n      this.mediaStream.getTracks().forEach((track) => track.stop())\n      this.mediaStream = null\n    }\n  }\n\n  startAudioCapture() {\n    if (this.mediaStream) {\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = true\n      }\n    }\n    this.isStreaming = true\n  }\n\n  stopAudioCapture() {\n    if (this.mediaStream) {\n      // Disable tracks rather than stopping them to allow re-enabling\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = false\n      }\n    }\n    this.isStreaming = false\n  }\n\n  private convertToPCM16(float32Array: Float32Array): ArrayBuffer {\n    const int16Array = new Int16Array(float32Array.length)\n    for (let i = 0; i < float32Array.length; i++) {\n      const sample = Math.max(-1, Math.min(1, float32Array[i] ?? 0))\n      int16Array[i] = sample * 0x7fff\n    }\n    return int16Array.buffer\n  }\n\n  private arrayBufferToBase64(buffer: ArrayBuffer): string {\n    const bytes = new Uint8Array(buffer)\n    const binary = String.fromCharCode(...bytes)\n    return btoa(binary)\n  }\n}\n\nexport class AudioPlayer {\n  private audioContext: AudioContext | null = null\n  private workletNode: AudioWorkletNode | null = null\n  private gainNode: GainNode | null = null\n  private analyser: AnalyserNode | null = null\n  private isInitialized = false\n  private volume = 1.0\n  private readonly sampleRate = 24000\n\n  get outputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get outputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get outputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get outputSampleRate() {\n    return this.sampleRate\n  }\n\n  async init() {\n    if (this.isInitialized) return\n\n    try {\n      // Create audio context at 24kHz to match Gemini\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      const workletBlob = new Blob([playbackWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create worklet node\n      this.workletNode = new AudioWorkletNode(\n        this.audioContext,\n        'pcm-processor',\n      )\n\n      // Create gain node for volume control\n      this.gainNode = this.audioContext.createGain()\n      this.gainNode.gain.value = this.volume\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect nodes\n      this.workletNode.connect(this.gainNode)\n      this.gainNode.connect(this.analyser)\n      this.analyser.connect(this.audioContext.destination)\n\n      this.isInitialized = true\n    } catch (error) {\n      // Release the audio context if initialization failed partway through.\n      this.destroy()\n      throw error\n    }\n  }\n\n  async play(pcmData: ArrayBuffer) {\n    if (!this.isInitialized) {\n      await this.init()\n    }\n\n    // Resume audio context if suspended\n    if (this.audioContext?.state === 'suspended') {\n      await this.audioContext.resume()\n    }\n\n    // Convert PCM16 LE to Float32\n    const inputArray = new Int16Array(pcmData)\n    const float32Data = new Float32Array(inputArray.length)\n    for (let i = 0; i < inputArray.length; i++) {\n      float32Data[i] = (inputArray[i] ?? 0) / 32768\n    }\n\n    // Send to worklet for playback\n    this.workletNode?.port.postMessage(float32Data)\n  }\n\n  /* Interrupt playback */\n  interrupt() {\n    if (this.workletNode) {\n      this.workletNode.port.postMessage('interrupt')\n    }\n  }\n\n  setVolume(volume: number) {\n    this.volume = Math.max(0, Math.min(1, volume))\n    if (this.gainNode) {\n      this.gainNode.gain.value = this.volume\n    }\n  }\n\n  destroy() {\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n    this.isInitialized = false\n  }\n}\n"],"mappings":";;;;AAKA,IAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8C3B,IAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+D5B,SAAS,eAAe,UAAgC;CACtD,MAAM,OAAO,IAAI,WAAW,SAAS,OAAO;CAC5C,SAAS,sBAAsB,IAAI;CAInC,IAAI,eAAe;CACnB,KAAK,MAAM,UAAU,MAAM;EACzB,MAAM,YAAY,KAAK,IAAI,SAAS,GAAG;EACvC,IAAI,YAAY,cACd,eAAe;CAEnB;CAKA,MAAM,aAAa,eAAe;CAClC,OAAO,KAAK,IAAI,GAAG,aAAa,GAAG;AACrC;AAEA,SAAgB,oBAAoB,QAA6B;CAC/D,MAAM,SAAS,KAAK,MAAM;CAE1B,OADc,WAAW,KAAK,SAAS,SAAS,KAAK,WAAW,CAAC,CAC1D,CAAA,CAAM;AACf;AAIA,IAAM,qCAAqB,IAAI,WAAW,IAAI;AAC9C,IAAM,uCAAsB,IAAI,WAAW,IAAI,EAAA,CAAE,KAAK,GAAG;AAEzD,IAAa,gBAAb,MAA2B;CACzB,eAA4C;CAC5C,eAAgD;CAChD,cAA0C;CAC1C,WAAwC;CACxC,cAAsB;CACtB,aAA8B;CAC9B,SAAmD;CAEnD,YAAY,QAA0B;EACpC,KAAK,SAAS;CAChB;CAEA,IAAI,aAAa;EACf,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,OAAO,eAAe,KAAK,QAAQ;CACrC;CAEA,IAAI,qBAAqB;EACvB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;EAC3D,KAAK,SAAS,qBAAqB,IAAI;EACvC,OAAO;CACT;CAEA,IAAI,sBAAsB;EACxB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;EACjD,KAAK,SAAS,sBAAsB,IAAI;EACxC,OAAO;CACT;CAEA,IAAI,kBAAkB;EACpB,OAAO,KAAK;CACd;CAEA,MAAM,QAAQ;EACZ,IAAI;GACF,MAAM,mBAA0C;IAC9C,YAAY,KAAK;IACjB,kBAAkB;IAClB,kBAAkB;IAClB,iBAAiB;GACnB;GAGA,KAAK,cAAc,MAAM,UAAU,aAAa,aAAa,EAC3D,OAAO,iBACT,CAAC;GAMD,KAHc,KAAK,YAAY,eAAe,CAAC,CAAC,EAC/B,EAAO,YAAY,EAAA,EAEtB,iBACZ,QAAQ,KAAK,2BAA2B;GAI1C,KAAK,eAAe,IAAI,aAAa,EACnC,YAAY,KAAK,WACnB,CAAC;GAED,IAAI,KAAK,aAAa,UAAU,aAC9B,MAAM,KAAK,aAAa,OAAO;GAGjC,MAAM,cAAc,IAAI,KAAK,CAAC,kBAAkB,GAAG,EACjD,MAAM,yBACR,CAAC;GACD,MAAM,aAAa,IAAI,gBAAgB,WAAW;GAGlD,MAAM,KAAK,aAAa,aAAa,UAAU,UAAU;GACzD,IAAI,gBAAgB,UAAU;GAG9B,KAAK,eAAe,IAAI,iBACtB,KAAK,cACL,yBACF;GAGA,KAAK,aAAa,KAAK,aAAa,UAAU;IAC5C,IAAI,CAAC,KAAK,aAAa;IAEvB,IAAI,MAAM,KAAK,SAAS,SAAS;KAC/B,MAAM,YAAY,MAAM,KAAK;KAC7B,MAAM,UAAU,KAAK,eAAe,SAAS;KAC7C,MAAM,cAAc,KAAK,oBAAoB,OAAO;KAGpD,IAAI,KAAK,QAAQ,iBACf,KAAK,OAAO,iBAAiB,WAAW;IAE5C;GACF;GAGA,KAAK,WAAW,KAAK,aAAa,eAAe;GACjD,KAAK,SAAS,UAAU;GACxB,KAAK,SAAS,wBAAwB;GAItC,KADoB,aAAa,wBAAwB,KAAK,WAC9D,CAAA,CAAO,QAAQ,KAAK,QAAQ;GAC5B,KAAK,SAAS,QAAQ,KAAK,YAAY;GAGvC,KAAK,cAAc;EACrB,SAAS,OAAO;GAEd,KAAK,KAAK;GACV,MAAM;EACR;CACF;CAEA,OAAO;EACL,KAAK,cAAc;EAEnB,IAAI,KAAK,cAAc;GACrB,KAAK,aAAa,WAAW;GAC7B,KAAK,aAAa,KAAK,MAAM;GAC7B,KAAK,eAAe;EACtB;EAEA,IAAI,KAAK,cAAc;GACrB,KAAU,aAAa,MAAM;GAC7B,KAAK,eAAe;EACtB;EAEA,IAAI,KAAK,aAAa;GACpB,KAAK,YAAY,UAAU,CAAC,CAAC,SAAS,UAAU,MAAM,KAAK,CAAC;GAC5D,KAAK,cAAc;EACrB;CACF;CAEA,oBAAoB;EAClB,IAAI,KAAK,aACP,KAAK,MAAM,SAAS,KAAK,YAAY,eAAe,GAClD,MAAM,UAAU;EAGpB,KAAK,cAAc;CACrB;CAEA,mBAAmB;EACjB,IAAI,KAAK,aAEP,KAAK,MAAM,SAAS,KAAK,YAAY,eAAe,GAClD,MAAM,UAAU;EAGpB,KAAK,cAAc;CACrB;CAEA,eAAuB,cAAyC;EAC9D,MAAM,aAAa,IAAI,WAAW,aAAa,MAAM;EACrD,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,QAAQ,KAAK;GAC5C,MAAM,SAAS,KAAK,IAAI,IAAI,KAAK,IAAI,GAAG,aAAa,MAAM,CAAC,CAAC;GAC7D,WAAW,KAAK,SAAS;EAC3B;EACA,OAAO,WAAW;CACpB;CAEA,oBAA4B,QAA6B;EACvD,MAAM,QAAQ,IAAI,WAAW,MAAM;EACnC,MAAM,SAAS,OAAO,aAAa,GAAG,KAAK;EAC3C,OAAO,KAAK,MAAM;CACpB;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB,eAA4C;CAC5C,cAA+C;CAC/C,WAAoC;CACpC,WAAwC;CACxC,gBAAwB;CACxB,SAAiB;CACjB,aAA8B;CAE9B,IAAI,cAAc;EAChB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,OAAO,eAAe,KAAK,QAAQ;CACrC;CAEA,IAAI,sBAAsB;EACxB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;EAC3D,KAAK,SAAS,qBAAqB,IAAI;EACvC,OAAO;CACT;CAEA,IAAI,uBAAuB;EACzB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;EACjD,KAAK,SAAS,sBAAsB,IAAI;EACxC,OAAO;CACT;CAEA,IAAI,mBAAmB;EACrB,OAAO,KAAK;CACd;CAEA,MAAM,OAAO;EACX,IAAI,KAAK,eAAe;EAExB,IAAI;GAEF,KAAK,eAAe,IAAI,aAAa,EACnC,YAAY,KAAK,WACnB,CAAC;GAED,MAAM,cAAc,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAClD,MAAM,yBACR,CAAC;GACD,MAAM,aAAa,IAAI,gBAAgB,WAAW;GAGlD,MAAM,KAAK,aAAa,aAAa,UAAU,UAAU;GACzD,IAAI,gBAAgB,UAAU;GAG9B,KAAK,cAAc,IAAI,iBACrB,KAAK,cACL,eACF;GAGA,KAAK,WAAW,KAAK,aAAa,WAAW;GAC7C,KAAK,SAAS,KAAK,QAAQ,KAAK;GAGhC,KAAK,WAAW,KAAK,aAAa,eAAe;GACjD,KAAK,SAAS,UAAU;GACxB,KAAK,SAAS,wBAAwB;GAGtC,KAAK,YAAY,QAAQ,KAAK,QAAQ;GACtC,KAAK,SAAS,QAAQ,KAAK,QAAQ;GACnC,KAAK,SAAS,QAAQ,KAAK,aAAa,WAAW;GAEnD,KAAK,gBAAgB;EACvB,SAAS,OAAO;GAEd,KAAK,QAAQ;GACb,MAAM;EACR;CACF;CAEA,MAAM,KAAK,SAAsB;EAC/B,IAAI,CAAC,KAAK,eACR,MAAM,KAAK,KAAK;EAIlB,IAAI,KAAK,cAAc,UAAU,aAC/B,MAAM,KAAK,aAAa,OAAO;EAIjC,MAAM,aAAa,IAAI,WAAW,OAAO;EACzC,MAAM,cAAc,IAAI,aAAa,WAAW,MAAM;EACtD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KACrC,YAAY,MAAM,WAAW,MAAM,KAAK;EAI1C,KAAK,aAAa,KAAK,YAAY,WAAW;CAChD;CAGA,YAAY;EACV,IAAI,KAAK,aACP,KAAK,YAAY,KAAK,YAAY,WAAW;CAEjD;CAEA,UAAU,QAAgB;EACxB,KAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;EAC7C,IAAI,KAAK,UACP,KAAK,SAAS,KAAK,QAAQ,KAAK;CAEpC;CAEA,UAAU;EACR,IAAI,KAAK,cAAc;GACrB,KAAU,aAAa,MAAM;GAC7B,KAAK,eAAe;EACtB;EACA,KAAK,gBAAgB;CACvB;AACF"}

package/dist/esm/tools/tool-converter.js

@@ -62,7 +62,6 @@ function convertToolsToProviderFormat(tools) {
 					required: []
 				}
 			});
-			break;
 	}
 	if (functionDeclarations.length > 0) result.push({ functionDeclarations });
 	return result;

package/dist/esm/tools/tool-converter.js.map

@@ -1 +1 @@
-{"version":3,"file":"tool-converter.js","names":[],"sources":["../../../src/tools/tool-converter.ts"],"sourcesContent":["import { convertCodeExecutionToolToAdapterFormat } from './code-execution-tool'\nimport { convertComputerUseToolToAdapterFormat } from './computer-use-tool'\nimport { convertFileSearchToolToAdapterFormat } from './file-search-tool'\nimport { convertGoogleMapsToolToAdapterFormat } from './google-maps-tool'\nimport { convertGoogleSearchRetrievalToolToAdapterFormat } from './google-search-retriveal-tool'\nimport { convertGoogleSearchToolToAdapterFormat } from './google-search-tool'\nimport { convertUrlContextToolToAdapterFormat } from './url-context-tool'\nimport type { Tool } from '@tanstack/ai'\nimport type { ToolUnion } from '@google/genai'\n\n/**\n * Converts standard Tool format to Gemini-specific tool format\n *\n * @param tools - Array of standard Tool objects\n * @returns Array of Gemini-specific tool definitions\n *\n * @example\n * ```typescript\n * const tools: Tool[] = [{\n *   name: \"get_weather\",\n *   description: \"Get weather for a location\",\n *   inputSchema: z.object({\n *     location: z.string()\n *   })\n * }];\n *\n * const geminiTools = convertToolsToProviderFormat(tools);\n * ```\n */\nexport function convertToolsToProviderFormat<TTool extends Tool>(\n  tools: Array<TTool> | undefined,\n): Array<ToolUnion> {\n  if (!tools || tools.length === 0) {\n    return []\n  }\n  const result: Array<ToolUnion> = []\n  const functionDeclarations: Array<{\n    name: string\n    description?: string\n    parameters?: any\n  }> = []\n\n  // Process each tool and group function declarations together\n  for (const tool of tools) {\n    const name = tool.name\n\n    switch (name) {\n      case 'code_execution':\n        result.push(convertCodeExecutionToolToAdapterFormat(tool))\n        break\n      case 'computer_use':\n        result.push(convertComputerUseToolToAdapterFormat(tool))\n        break\n      case 'file_search':\n        result.push(convertFileSearchToolToAdapterFormat(tool))\n        break\n      case 'google_maps':\n        result.push(convertGoogleMapsToolToAdapterFormat(tool))\n        break\n      case 'google_search_retrieval':\n        result.push(convertGoogleSearchRetrievalToolToAdapterFormat(tool))\n        break\n      case 'google_search':\n        result.push(convertGoogleSearchToolToAdapterFormat(tool))\n        break\n      case 'url_context':\n        result.push(convertUrlContextToolToAdapterFormat(tool))\n        break\n      default:\n        // Collect function declarations to group together\n        // Description is required for Gemini function declarations\n        if (!tool.description) {\n          throw new Error(\n            `Tool ${tool.name} requires a description for Gemini adapter`,\n          )\n        }\n\n        // Tool schemas are already converted to JSON Schema in the ai layer\n        functionDeclarations.push({\n          name: tool.name,\n          description: tool.description,\n          parameters: tool.inputSchema ?? {\n            type: 'object',\n            properties: {},\n            required: [],\n          },\n        })\n        break\n    }\n  }\n\n  // If we have function declarations, add them as a single tool\n  if (functionDeclarations.length > 0) {\n    result.push({\n      functionDeclarations: functionDeclarations,\n    })\n  }\n\n  return result\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,6BACd,OACkB;CAClB,IAAI,CAAC,SAAS,MAAM,WAAW,GAC7B,OAAO,CAAC;CAEV,MAAM,SAA2B,CAAC;CAClC,MAAM,uBAID,CAAC;CAGN,KAAK,MAAM,QAAQ,OAGjB,QAFa,KAAK,MAElB;EACE,KAAK;GACH,OAAO,KAAK,wCAAwC,IAAI,CAAC;GACzD;EACF,KAAK;GACH,OAAO,KAAK,sCAAsC,IAAI,CAAC;GACvD;EACF,KAAK;GACH,OAAO,KAAK,qCAAqC,IAAI,CAAC;GACtD;EACF,KAAK;GACH,OAAO,KAAK,qCAAqC,IAAI,CAAC;GACtD;EACF,KAAK;GACH,OAAO,KAAK,gDAAgD,IAAI,CAAC;GACjE;EACF,KAAK;GACH,OAAO,KAAK,uCAAuC,IAAI,CAAC;GACxD;EACF,KAAK;GACH,OAAO,KAAK,qCAAqC,IAAI,CAAC;GACtD;EACF;GAGE,IAAI,CAAC,KAAK,aACR,MAAM,IAAI,MACR,QAAQ,KAAK,KAAK,2CACpB;GAIF,qBAAqB,KAAK;IACxB,MAAM,KAAK;IACX,aAAa,KAAK;IAClB,YAAY,KAAK,eAAe;KAC9B,MAAM;KACN,YAAY,CAAC;KACb,UAAU,CAAC;IACb;GACF,CAAC;GACD;CACJ;CAIF,IAAI,qBAAqB,SAAS,GAChC,OAAO,KAAK,EACY,qBACxB,CAAC;CAGH,OAAO;AACT"}
+{"version":3,"file":"tool-converter.js","names":[],"sources":["../../../src/tools/tool-converter.ts"],"sourcesContent":["import { convertCodeExecutionToolToAdapterFormat } from './code-execution-tool'\nimport { convertComputerUseToolToAdapterFormat } from './computer-use-tool'\nimport { convertFileSearchToolToAdapterFormat } from './file-search-tool'\nimport { convertGoogleMapsToolToAdapterFormat } from './google-maps-tool'\nimport { convertGoogleSearchRetrievalToolToAdapterFormat } from './google-search-retriveal-tool'\nimport { convertGoogleSearchToolToAdapterFormat } from './google-search-tool'\nimport { convertUrlContextToolToAdapterFormat } from './url-context-tool'\nimport type { Tool } from '@tanstack/ai'\nimport type { ToolUnion } from '@google/genai'\n\n/**\n * Converts standard Tool format to Gemini-specific tool format\n *\n * @param tools - Array of standard Tool objects\n * @returns Array of Gemini-specific tool definitions\n *\n * @example\n * ```typescript\n * const tools: Tool[] = [{\n *   name: \"get_weather\",\n *   description: \"Get weather for a location\",\n *   inputSchema: z.object({\n *     location: z.string()\n *   })\n * }];\n *\n * const geminiTools = convertToolsToProviderFormat(tools);\n * ```\n */\nexport function convertToolsToProviderFormat<TTool extends Tool>(\n  tools: Array<TTool> | undefined,\n): Array<ToolUnion> {\n  if (!tools || tools.length === 0) {\n    return []\n  }\n  const result: Array<ToolUnion> = []\n  const functionDeclarations: Array<{\n    name: string\n    description?: string\n    parameters?: any\n  }> = []\n\n  // Process each tool and group function declarations together\n  for (const tool of tools) {\n    const name = tool.name\n\n    switch (name) {\n      case 'code_execution':\n        result.push(convertCodeExecutionToolToAdapterFormat(tool))\n        break\n      case 'computer_use':\n        result.push(convertComputerUseToolToAdapterFormat(tool))\n        break\n      case 'file_search':\n        result.push(convertFileSearchToolToAdapterFormat(tool))\n        break\n      case 'google_maps':\n        result.push(convertGoogleMapsToolToAdapterFormat(tool))\n        break\n      case 'google_search_retrieval':\n        result.push(convertGoogleSearchRetrievalToolToAdapterFormat(tool))\n        break\n      case 'google_search':\n        result.push(convertGoogleSearchToolToAdapterFormat(tool))\n        break\n      case 'url_context':\n        result.push(convertUrlContextToolToAdapterFormat(tool))\n        break\n      default:\n        // Collect function declarations to group together\n        // Description is required for Gemini function declarations\n        if (!tool.description) {\n          throw new Error(\n            `Tool ${tool.name} requires a description for Gemini adapter`,\n          )\n        }\n\n        // Tool schemas are already converted to JSON Schema in the ai layer\n        functionDeclarations.push({\n          name: tool.name,\n          description: tool.description,\n          parameters: tool.inputSchema ?? {\n            type: 'object',\n            properties: {},\n            required: [],\n          },\n        })\n        break\n    }\n  }\n\n  // If we have function declarations, add them as a single tool\n  if (functionDeclarations.length > 0) {\n    result.push({\n      functionDeclarations: functionDeclarations,\n    })\n  }\n\n  return result\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,6BACd,OACkB;CAClB,IAAI,CAAC,SAAS,MAAM,WAAW,GAC7B,OAAO,CAAC;CAEV,MAAM,SAA2B,CAAC;CAClC,MAAM,uBAID,CAAC;CAGN,KAAK,MAAM,QAAQ,OAGjB,QAFa,KAAK,MAElB;EACE,KAAK;GACH,OAAO,KAAK,wCAAwC,IAAI,CAAC;GACzD;EACF,KAAK;GACH,OAAO,KAAK,sCAAsC,IAAI,CAAC;GACvD;EACF,KAAK;GACH,OAAO,KAAK,qCAAqC,IAAI,CAAC;GACtD;EACF,KAAK;GACH,OAAO,KAAK,qCAAqC,IAAI,CAAC;GACtD;EACF,KAAK;GACH,OAAO,KAAK,gDAAgD,IAAI,CAAC;GACjE;EACF,KAAK;GACH,OAAO,KAAK,uCAAuC,IAAI,CAAC;GACxD;EACF,KAAK;GACH,OAAO,KAAK,qCAAqC,IAAI,CAAC;GACtD;EACF;GAGE,IAAI,CAAC,KAAK,aACR,MAAM,IAAI,MACR,QAAQ,KAAK,KAAK,2CACpB;GAIF,qBAAqB,KAAK;IACxB,MAAM,KAAK;IACX,aAAa,KAAK;IAClB,YAAY,KAAK,eAAe;KAC9B,MAAM;KACN,YAAY,CAAC;KACb,UAAU,CAAC;IACb;GACF,CAAC;CAEL;CAIF,IAAI,qBAAqB,SAAS,GAChC,OAAO,KAAK,EACY,qBACxB,CAAC;CAGH,OAAO;AACT"}

package/dist/esm/usage.js

@@ -25,9 +25,7 @@ function flattenModalityTokenCounts(modalities) {
 			case "VIDEO":
 				result.videoTokens = (result.videoTokens ?? 0) + count;
 				break;
-			case "DOCUMENT":
-				result.documentTokens = (result.documentTokens ?? 0) + count;
-				break;
+			case "DOCUMENT": result.documentTokens = (result.documentTokens ?? 0) + count;
 		}
 	}
 	return result;

package/dist/esm/usage.js.map

@@ -1 +1 @@
-{"version":3,"file":"usage.js","names":[],"sources":["../../src/usage.ts"],"sourcesContent":["import { buildBaseUsage } from '@tanstack/ai'\nimport type { TokenUsage } from '@tanstack/ai'\nimport type {\n  GenerateContentResponseUsageMetadata,\n  ModalityTokenCount,\n} from '@google/genai'\n\n/**\n * Flattened modality token counts for normalized usage reporting.\n * Maps Gemini's ModalityTokenCount array to individual fields.\n */\nexport interface FlattenedModalityTokens {\n  /** Text tokens */\n  textTokens?: number\n  /** Image tokens */\n  imageTokens?: number\n  /** Audio tokens */\n  audioTokens?: number\n  /** Video tokens */\n  videoTokens?: number\n  /** Document tokens (e.g. PDF inputs) */\n  documentTokens?: number\n}\n\n/**\n * Flattens Gemini's ModalityTokenCount array into individual token fields.\n * Extracts TEXT, IMAGE, AUDIO, VIDEO, DOCUMENT modality counts into a\n * normalized structure.\n */\nexport function flattenModalityTokenCounts(\n  modalities?: Array<ModalityTokenCount>,\n): FlattenedModalityTokens {\n  if (!modalities || modalities.length === 0) {\n    return {}\n  }\n\n  const result: FlattenedModalityTokens = {}\n\n  for (const item of modalities) {\n    if (!item.modality || item.tokenCount === undefined) {\n      continue\n    }\n\n    const modality = item.modality.toUpperCase()\n    const count = item.tokenCount\n\n    switch (modality) {\n      case 'TEXT':\n        result.textTokens = (result.textTokens ?? 0) + count\n        break\n      case 'IMAGE':\n        result.imageTokens = (result.imageTokens ?? 0) + count\n        break\n      case 'AUDIO':\n        result.audioTokens = (result.audioTokens ?? 0) + count\n        break\n      case 'VIDEO':\n        result.videoTokens = (result.videoTokens ?? 0) + count\n        break\n      case 'DOCUMENT':\n        result.documentTokens = (result.documentTokens ?? 0) + count\n        break\n    }\n  }\n\n  return result\n}\n\n/**\n * Checks if a FlattenedModalityTokens object has any values set.\n */\nexport function hasModalityTokens(tokens: FlattenedModalityTokens): boolean {\n  return (\n    tokens.textTokens !== undefined ||\n    tokens.imageTokens !== undefined ||\n    tokens.audioTokens !== undefined ||\n    tokens.videoTokens !== undefined ||\n    tokens.documentTokens !== undefined\n  )\n}\n\n/**\n * Gemini-specific provider usage details.\n * These fields are unique to Gemini and placed in providerUsageDetails.\n */\nexport type GeminiProviderUsageDetails = {\n  /**\n   * The traffic type for this request.\n   * Can indicate whether request was handled by different service tiers.\n   */\n  trafficType?: string\n  /**\n   * Number of tokens in the results from tool executions,\n   * which are provided back to the model as input.\n   */\n  toolUsePromptTokenCount?: number\n  /**\n   * Detailed breakdown by modality of the token counts from\n   * the results of tool executions.\n   */\n  toolUsePromptTokensDetails?: Array<{\n    modality: string\n    tokenCount: number\n  }>\n  /**\n   * Detailed breakdown of cache tokens by modality.\n   * More granular than the normalized cachedTokens field.\n   */\n  cacheTokensDetails?: Array<{\n    modality: string\n    tokenCount: number\n  }>\n}\n\n/**\n * Build normalized TokenUsage from Gemini's usageMetadata.\n * Handles modality breakdowns and thinking tokens. Returns `undefined` when the\n * provider reported no usage metadata, so callers omit the field rather than\n * fabricating zeroed totals.\n */\nexport function buildGeminiUsage(\n  usageMetadata: GenerateContentResponseUsageMetadata | undefined | null,\n): TokenUsage<GeminiProviderUsageDetails> | undefined {\n  if (!usageMetadata) return undefined\n\n  const promptTokens = usageMetadata.promptTokenCount ?? 0\n  const completionTokens = usageMetadata.candidatesTokenCount ?? 0\n\n  const result = buildBaseUsage<GeminiProviderUsageDetails>({\n    promptTokens: promptTokens,\n    completionTokens: completionTokens,\n    totalTokens:\n      usageMetadata.totalTokenCount ?? promptTokens + completionTokens,\n  })\n\n  // Add prompt token details\n  // Flatten modality breakdown for prompt\n  const promptModalities = flattenModalityTokenCounts(\n    usageMetadata.promptTokensDetails,\n  )\n  const cachedTokens = usageMetadata.cachedContentTokenCount\n\n  const promptTokensDetails = {\n    ...(hasModalityTokens(promptModalities) ? promptModalities : {}),\n    ...(cachedTokens !== undefined && cachedTokens > 0 ? { cachedTokens } : {}),\n  }\n\n  // Add completion token details\n  // Flatten modality breakdown for candidates (output)\n  const completionModalities = flattenModalityTokenCounts(\n    usageMetadata.candidatesTokensDetails,\n  )\n  const thoughtsTokens = usageMetadata.thoughtsTokenCount\n\n  const completionTokensDetails = {\n    ...(hasModalityTokens(completionModalities) ? completionModalities : {}),\n    // Map thoughtsTokenCount to reasoningTokens for consistency with OpenAI\n    ...(thoughtsTokens !== undefined && thoughtsTokens > 0\n      ? { reasoningTokens: thoughtsTokens }\n      : {}),\n  }\n\n  // Add provider-specific details\n  const providerDetails: GeminiProviderUsageDetails = {\n    ...(usageMetadata.trafficType\n      ? { trafficType: usageMetadata.trafficType }\n      : {}),\n    ...(usageMetadata.toolUsePromptTokenCount !== undefined &&\n    usageMetadata.toolUsePromptTokenCount > 0\n      ? { toolUsePromptTokenCount: usageMetadata.toolUsePromptTokenCount }\n      : {}),\n    ...(usageMetadata.toolUsePromptTokensDetails &&\n    usageMetadata.toolUsePromptTokensDetails.length > 0\n      ? {\n          toolUsePromptTokensDetails:\n            usageMetadata.toolUsePromptTokensDetails.map((item) => ({\n              modality: item.modality || 'UNKNOWN',\n              tokenCount: item.tokenCount ?? 0,\n            })),\n        }\n      : {}),\n    ...(usageMetadata.cacheTokensDetails &&\n    usageMetadata.cacheTokensDetails.length > 0\n      ? {\n          cacheTokensDetails: usageMetadata.cacheTokensDetails.map((item) => ({\n            modality: item.modality || 'UNKNOWN',\n            tokenCount: item.tokenCount ?? 0,\n          })),\n        }\n      : {}),\n  }\n\n  // Add prompt token details if available\n  if (Object.keys(promptTokensDetails).length > 0) {\n    result.promptTokensDetails = promptTokensDetails\n  }\n  // Add provider details if available\n  if (Object.keys(providerDetails).length > 0) {\n    result.providerUsageDetails = providerDetails\n  }\n  // Add completion token details if available\n  if (Object.keys(completionTokensDetails).length > 0) {\n    result.completionTokensDetails = completionTokensDetails\n  }\n\n  return result\n}\n"],"mappings":";;;;;;;AA6BA,SAAgB,2BACd,YACyB;CACzB,IAAI,CAAC,cAAc,WAAW,WAAW,GACvC,OAAO,CAAC;CAGV,MAAM,SAAkC,CAAC;CAEzC,KAAK,MAAM,QAAQ,YAAY;EAC7B,IAAI,CAAC,KAAK,YAAY,KAAK,eAAe,KAAA,GACxC;EAGF,MAAM,WAAW,KAAK,SAAS,YAAY;EAC3C,MAAM,QAAQ,KAAK;EAEnB,QAAQ,UAAR;GACE,KAAK;IACH,OAAO,cAAc,OAAO,cAAc,KAAK;IAC/C;GACF,KAAK;IACH,OAAO,eAAe,OAAO,eAAe,KAAK;IACjD;GACF,KAAK;IACH,OAAO,eAAe,OAAO,eAAe,KAAK;IACjD;GACF,KAAK;IACH,OAAO,eAAe,OAAO,eAAe,KAAK;IACjD;GACF,KAAK;IACH,OAAO,kBAAkB,OAAO,kBAAkB,KAAK;IACvD;EACJ;CACF;CAEA,OAAO;AACT;;;;AAKA,SAAgB,kBAAkB,QAA0C;CAC1E,OACE,OAAO,eAAe,KAAA,KACtB,OAAO,gBAAgB,KAAA,KACvB,OAAO,gBAAgB,KAAA,KACvB,OAAO,gBAAgB,KAAA,KACvB,OAAO,mBAAmB,KAAA;AAE9B;;;;;;;AAyCA,SAAgB,iBACd,eACoD;CACpD,IAAI,CAAC,eAAe,OAAO,KAAA;CAE3B,MAAM,eAAe,cAAc,oBAAoB;CACvD,MAAM,mBAAmB,cAAc,wBAAwB;CAE/D,MAAM,SAAS,eAA2C;EAC1C;EACI;EAClB,aACE,cAAc,mBAAmB,eAAe;CACpD,CAAC;CAID,MAAM,mBAAmB,2BACvB,cAAc,mBAChB;CACA,MAAM,eAAe,cAAc;CAEnC,MAAM,sBAAsB;EAC1B,GAAI,kBAAkB,gBAAgB,IAAI,mBAAmB,CAAC;EAC9D,GAAI,iBAAiB,KAAA,KAAa,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;CAC3E;CAIA,MAAM,uBAAuB,2BAC3B,cAAc,uBAChB;CACA,MAAM,iBAAiB,cAAc;CAErC,MAAM,0BAA0B;EAC9B,GAAI,kBAAkB,oBAAoB,IAAI,uBAAuB,CAAC;EAEtE,GAAI,mBAAmB,KAAA,KAAa,iBAAiB,IACjD,EAAE,iBAAiB,eAAe,IAClC,CAAC;CACP;CAGA,MAAM,kBAA8C;EAClD,GAAI,cAAc,cACd,EAAE,aAAa,cAAc,YAAY,IACzC,CAAC;EACL,GAAI,cAAc,4BAA4B,KAAA,KAC9C,cAAc,0BAA0B,IACpC,EAAE,yBAAyB,cAAc,wBAAwB,IACjE,CAAC;EACL,GAAI,cAAc,8BAClB,cAAc,2BAA2B,SAAS,IAC9C,EACE,4BACE,cAAc,2BAA2B,KAAK,UAAU;GACtD,UAAU,KAAK,YAAY;GAC3B,YAAY,KAAK,cAAc;EACjC,EAAE,EACN,IACA,CAAC;EACL,GAAI,cAAc,sBAClB,cAAc,mBAAmB,SAAS,IACtC,EACE,oBAAoB,cAAc,mBAAmB,KAAK,UAAU;GAClE,UAAU,KAAK,YAAY;GAC3B,YAAY,KAAK,cAAc;EACjC,EAAE,EACJ,IACA,CAAC;CACP;CAGA,IAAI,OAAO,KAAK,mBAAmB,CAAC,CAAC,SAAS,GAC5C,OAAO,sBAAsB;CAG/B,IAAI,OAAO,KAAK,eAAe,CAAC,CAAC,SAAS,GACxC,OAAO,uBAAuB;CAGhC,IAAI,OAAO,KAAK,uBAAuB,CAAC,CAAC,SAAS,GAChD,OAAO,0BAA0B;CAGnC,OAAO;AACT"}
+{"version":3,"file":"usage.js","names":[],"sources":["../../src/usage.ts"],"sourcesContent":["import { buildBaseUsage } from '@tanstack/ai'\nimport type { TokenUsage } from '@tanstack/ai'\nimport type {\n  GenerateContentResponseUsageMetadata,\n  ModalityTokenCount,\n} from '@google/genai'\n\n/**\n * Flattened modality token counts for normalized usage reporting.\n * Maps Gemini's ModalityTokenCount array to individual fields.\n */\nexport interface FlattenedModalityTokens {\n  /** Text tokens */\n  textTokens?: number\n  /** Image tokens */\n  imageTokens?: number\n  /** Audio tokens */\n  audioTokens?: number\n  /** Video tokens */\n  videoTokens?: number\n  /** Document tokens (e.g. PDF inputs) */\n  documentTokens?: number\n}\n\n/**\n * Flattens Gemini's ModalityTokenCount array into individual token fields.\n * Extracts TEXT, IMAGE, AUDIO, VIDEO, DOCUMENT modality counts into a\n * normalized structure.\n */\nexport function flattenModalityTokenCounts(\n  modalities?: Array<ModalityTokenCount>,\n): FlattenedModalityTokens {\n  if (!modalities || modalities.length === 0) {\n    return {}\n  }\n\n  const result: FlattenedModalityTokens = {}\n\n  for (const item of modalities) {\n    if (!item.modality || item.tokenCount === undefined) {\n      continue\n    }\n\n    const modality = item.modality.toUpperCase()\n    const count = item.tokenCount\n\n    switch (modality) {\n      case 'TEXT':\n        result.textTokens = (result.textTokens ?? 0) + count\n        break\n      case 'IMAGE':\n        result.imageTokens = (result.imageTokens ?? 0) + count\n        break\n      case 'AUDIO':\n        result.audioTokens = (result.audioTokens ?? 0) + count\n        break\n      case 'VIDEO':\n        result.videoTokens = (result.videoTokens ?? 0) + count\n        break\n      case 'DOCUMENT':\n        result.documentTokens = (result.documentTokens ?? 0) + count\n        break\n    }\n  }\n\n  return result\n}\n\n/**\n * Checks if a FlattenedModalityTokens object has any values set.\n */\nexport function hasModalityTokens(tokens: FlattenedModalityTokens): boolean {\n  return (\n    tokens.textTokens !== undefined ||\n    tokens.imageTokens !== undefined ||\n    tokens.audioTokens !== undefined ||\n    tokens.videoTokens !== undefined ||\n    tokens.documentTokens !== undefined\n  )\n}\n\n/**\n * Gemini-specific provider usage details.\n * These fields are unique to Gemini and placed in providerUsageDetails.\n */\nexport type GeminiProviderUsageDetails = {\n  /**\n   * The traffic type for this request.\n   * Can indicate whether request was handled by different service tiers.\n   */\n  trafficType?: string\n  /**\n   * Number of tokens in the results from tool executions,\n   * which are provided back to the model as input.\n   */\n  toolUsePromptTokenCount?: number\n  /**\n   * Detailed breakdown by modality of the token counts from\n   * the results of tool executions.\n   */\n  toolUsePromptTokensDetails?: Array<{\n    modality: string\n    tokenCount: number\n  }>\n  /**\n   * Detailed breakdown of cache tokens by modality.\n   * More granular than the normalized cachedTokens field.\n   */\n  cacheTokensDetails?: Array<{\n    modality: string\n    tokenCount: number\n  }>\n}\n\n/**\n * Build normalized TokenUsage from Gemini's usageMetadata.\n * Handles modality breakdowns and thinking tokens. Returns `undefined` when the\n * provider reported no usage metadata, so callers omit the field rather than\n * fabricating zeroed totals.\n */\nexport function buildGeminiUsage(\n  usageMetadata: GenerateContentResponseUsageMetadata | undefined | null,\n): TokenUsage<GeminiProviderUsageDetails> | undefined {\n  if (!usageMetadata) return undefined\n\n  const promptTokens = usageMetadata.promptTokenCount ?? 0\n  const completionTokens = usageMetadata.candidatesTokenCount ?? 0\n\n  const result = buildBaseUsage<GeminiProviderUsageDetails>({\n    promptTokens: promptTokens,\n    completionTokens: completionTokens,\n    totalTokens:\n      usageMetadata.totalTokenCount ?? promptTokens + completionTokens,\n  })\n\n  // Add prompt token details\n  // Flatten modality breakdown for prompt\n  const promptModalities = flattenModalityTokenCounts(\n    usageMetadata.promptTokensDetails,\n  )\n  const cachedTokens = usageMetadata.cachedContentTokenCount\n\n  const promptTokensDetails = {\n    ...(hasModalityTokens(promptModalities) ? promptModalities : {}),\n    ...(cachedTokens !== undefined && cachedTokens > 0 ? { cachedTokens } : {}),\n  }\n\n  // Add completion token details\n  // Flatten modality breakdown for candidates (output)\n  const completionModalities = flattenModalityTokenCounts(\n    usageMetadata.candidatesTokensDetails,\n  )\n  const thoughtsTokens = usageMetadata.thoughtsTokenCount\n\n  const completionTokensDetails = {\n    ...(hasModalityTokens(completionModalities) ? completionModalities : {}),\n    // Map thoughtsTokenCount to reasoningTokens for consistency with OpenAI\n    ...(thoughtsTokens !== undefined && thoughtsTokens > 0\n      ? { reasoningTokens: thoughtsTokens }\n      : {}),\n  }\n\n  // Add provider-specific details\n  const providerDetails: GeminiProviderUsageDetails = {\n    ...(usageMetadata.trafficType\n      ? { trafficType: usageMetadata.trafficType }\n      : {}),\n    ...(usageMetadata.toolUsePromptTokenCount !== undefined &&\n    usageMetadata.toolUsePromptTokenCount > 0\n      ? { toolUsePromptTokenCount: usageMetadata.toolUsePromptTokenCount }\n      : {}),\n    ...(usageMetadata.toolUsePromptTokensDetails &&\n    usageMetadata.toolUsePromptTokensDetails.length > 0\n      ? {\n          toolUsePromptTokensDetails:\n            usageMetadata.toolUsePromptTokensDetails.map((item) => ({\n              modality: item.modality || 'UNKNOWN',\n              tokenCount: item.tokenCount ?? 0,\n            })),\n        }\n      : {}),\n    ...(usageMetadata.cacheTokensDetails &&\n    usageMetadata.cacheTokensDetails.length > 0\n      ? {\n          cacheTokensDetails: usageMetadata.cacheTokensDetails.map((item) => ({\n            modality: item.modality || 'UNKNOWN',\n            tokenCount: item.tokenCount ?? 0,\n          })),\n        }\n      : {}),\n  }\n\n  // Add prompt token details if available\n  if (Object.keys(promptTokensDetails).length > 0) {\n    result.promptTokensDetails = promptTokensDetails\n  }\n  // Add provider details if available\n  if (Object.keys(providerDetails).length > 0) {\n    result.providerUsageDetails = providerDetails\n  }\n  // Add completion token details if available\n  if (Object.keys(completionTokensDetails).length > 0) {\n    result.completionTokensDetails = completionTokensDetails\n  }\n\n  return result\n}\n"],"mappings":";;;;;;;AA6BA,SAAgB,2BACd,YACyB;CACzB,IAAI,CAAC,cAAc,WAAW,WAAW,GACvC,OAAO,CAAC;CAGV,MAAM,SAAkC,CAAC;CAEzC,KAAK,MAAM,QAAQ,YAAY;EAC7B,IAAI,CAAC,KAAK,YAAY,KAAK,eAAe,KAAA,GACxC;EAGF,MAAM,WAAW,KAAK,SAAS,YAAY;EAC3C,MAAM,QAAQ,KAAK;EAEnB,QAAQ,UAAR;GACE,KAAK;IACH,OAAO,cAAc,OAAO,cAAc,KAAK;IAC/C;GACF,KAAK;IACH,OAAO,eAAe,OAAO,eAAe,KAAK;IACjD;GACF,KAAK;IACH,OAAO,eAAe,OAAO,eAAe,KAAK;IACjD;GACF,KAAK;IACH,OAAO,eAAe,OAAO,eAAe,KAAK;IACjD;GACF,KAAK,YACH,OAAO,kBAAkB,OAAO,kBAAkB,KAAK;EAE3D;CACF;CAEA,OAAO;AACT;;;;AAKA,SAAgB,kBAAkB,QAA0C;CAC1E,OACE,OAAO,eAAe,KAAA,KACtB,OAAO,gBAAgB,KAAA,KACvB,OAAO,gBAAgB,KAAA,KACvB,OAAO,gBAAgB,KAAA,KACvB,OAAO,mBAAmB,KAAA;AAE9B;;;;;;;AAyCA,SAAgB,iBACd,eACoD;CACpD,IAAI,CAAC,eAAe,OAAO,KAAA;CAE3B,MAAM,eAAe,cAAc,oBAAoB;CACvD,MAAM,mBAAmB,cAAc,wBAAwB;CAE/D,MAAM,SAAS,eAA2C;EAC1C;EACI;EAClB,aACE,cAAc,mBAAmB,eAAe;CACpD,CAAC;CAID,MAAM,mBAAmB,2BACvB,cAAc,mBAChB;CACA,MAAM,eAAe,cAAc;CAEnC,MAAM,sBAAsB;EAC1B,GAAI,kBAAkB,gBAAgB,IAAI,mBAAmB,CAAC;EAC9D,GAAI,iBAAiB,KAAA,KAAa,eAAe,IAAI,EAAE,aAAa,IAAI,CAAC;CAC3E;CAIA,MAAM,uBAAuB,2BAC3B,cAAc,uBAChB;CACA,MAAM,iBAAiB,cAAc;CAErC,MAAM,0BAA0B;EAC9B,GAAI,kBAAkB,oBAAoB,IAAI,uBAAuB,CAAC;EAEtE,GAAI,mBAAmB,KAAA,KAAa,iBAAiB,IACjD,EAAE,iBAAiB,eAAe,IAClC,CAAC;CACP;CAGA,MAAM,kBAA8C;EAClD,GAAI,cAAc,cACd,EAAE,aAAa,cAAc,YAAY,IACzC,CAAC;EACL,GAAI,cAAc,4BAA4B,KAAA,KAC9C,cAAc,0BAA0B,IACpC,EAAE,yBAAyB,cAAc,wBAAwB,IACjE,CAAC;EACL,GAAI,cAAc,8BAClB,cAAc,2BAA2B,SAAS,IAC9C,EACE,4BACE,cAAc,2BAA2B,KAAK,UAAU;GACtD,UAAU,KAAK,YAAY;GAC3B,YAAY,KAAK,cAAc;EACjC,EAAE,EACN,IACA,CAAC;EACL,GAAI,cAAc,sBAClB,cAAc,mBAAmB,SAAS,IACtC,EACE,oBAAoB,cAAc,mBAAmB,KAAK,UAAU;GAClE,UAAU,KAAK,YAAY;GAC3B,YAAY,KAAK,cAAc;EACjC,EAAE,EACJ,IACA,CAAC;CACP;CAGA,IAAI,OAAO,KAAK,mBAAmB,CAAC,CAAC,SAAS,GAC5C,OAAO,sBAAsB;CAG/B,IAAI,OAAO,KAAK,eAAe,CAAC,CAAC,SAAS,GACxC,OAAO,uBAAuB;CAGhC,IAAI,OAAO,KAAK,uBAAuB,CAAC,CAAC,SAAS,GAChD,OAAO,0BAA0B;CAGnC,OAAO;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tanstack/ai-gemini",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "Google Gemini adapter for TanStack AI chat, images, speech, audio generation, and structured outputs.",
   "author": "Tanner Linsley",
   "license": "MIT",
@@ -59,13 +59,13 @@
     "@tanstack/ai-utils": "^0.4.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "^0.44.0"
+    "@tanstack/ai": "^0.45.0"
   },
   "devDependencies": {
-    "@vitest/coverage-v8": "4.0.14",
-    "vite": "^8.1.4",
+    "@vitest/coverage-v8": "4.1.10",
+    "vite": "^8.2.1",
     "zod": "^4.2.0",
-    "@tanstack/ai": "0.44.0"
+    "@tanstack/ai": "0.45.0"
   },
   "scripts": {
     "build": "vite build",

package/src/adapters/image.ts

@@ -7,18 +7,20 @@ import {
 } from '../utils'
 import { buildGeminiUsage } from '../usage'
 import {
+  isGeminiNativeImageModel,
   parseNativeImageSize,
   sizeToAspectRatio,
   validateImageSize,
   validateNumberOfImages,
   validatePrompt,
 } from '../image/image-provider-options'
-import type { GEMINI_IMAGE_MODELS } from '../model-meta'
+import type { GeminiImageModels } from '../model-meta'
 import type {
+  GeminiAnyImageProviderOptions,
   GeminiImageModelInputModalitiesByName,
   GeminiImageModelProviderOptionsByName,
   GeminiImageModelSizeByName,
-  GeminiImageProviderOptions,
+  GeminiNativeImageProviderOptions,
 } from '../image/image-provider-options'
 import type {
   GeneratedImage,
@@ -35,6 +37,7 @@ import type {
   GenerateImagesConfig,
   GenerateImagesResponse,
   GoogleGenAI,
+  ImageConfig,
   Part,
 } from '@google/genai'
 import type { GeminiClientConfig } from '../utils/client'
@@ -45,7 +48,7 @@ import type { GeminiClientConfig } from '../utils/client'
 export interface GeminiImageConfig extends GeminiClientConfig {}
 
 /** Model type for Gemini Image */
-export type GeminiImageModel = (typeof GEMINI_IMAGE_MODELS)[number]
+export type GeminiImageModel = GeminiImageModels
 
 /**
  * Gemini Image Generation Adapter
@@ -65,7 +68,7 @@ export class GeminiImageAdapter<
   TModel extends GeminiImageModel,
 > extends BaseImageAdapter<
   TModel,
-  GeminiImageProviderOptions,
+  GeminiAnyImageProviderOptions,
   GeminiImageModelProviderOptionsByName,
   GeminiImageModelSizeByName,
   GeminiImageModelInputModalitiesByName
@@ -75,7 +78,7 @@ export class GeminiImageAdapter<
 
   // Type-only property - never assigned at runtime
   declare '~types': {
-    providerOptions: GeminiImageProviderOptions
+    providerOptions: GeminiAnyImageProviderOptions
     modelProviderOptionsByName: GeminiImageModelProviderOptionsByName
     modelSizeByName: GeminiImageModelSizeByName
     modelInputModalitiesByName: GeminiImageModelInputModalitiesByName
@@ -89,7 +92,7 @@ export class GeminiImageAdapter<
   }
 
   async generateImages(
-    options: ImageGenerationOptions<GeminiImageProviderOptions>,
+    options: ImageGenerationOptions<GeminiAnyImageProviderOptions>,
   ): Promise<ImageGenerationResult> {
     const { model, logger } = options
 
@@ -121,7 +124,7 @@ export class GeminiImageAdapter<
         )
       }
 
-      if (this.isGeminiImageModel(model)) {
+      if (isGeminiNativeImageModel(model)) {
         return await this.generateWithGeminiApi(options, resolved)
       }
 
@@ -155,29 +158,40 @@ export class GeminiImageAdapter<
     }
   }
 
-  private isGeminiImageModel(model: string): boolean {
-    return model.startsWith('gemini-')
-  }
-
   private async generateWithGeminiApi(
-    options: ImageGenerationOptions<GeminiImageProviderOptions>,
+    options: ImageGenerationOptions<GeminiNativeImageProviderOptions>,
     resolved: ResolvedMediaPrompt,
   ): Promise<ImageGenerationResult> {
     const { model, size, numberOfImages, modelOptions } = options
 
     const parsedSize = size ? parseNativeImageSize(size) : undefined
 
-    // GeminiImageProviderOptions is Imagen-shaped — most fields
-    // (personGeneration, safetyFilterLevel, addWatermark, outputMimeType,
-    // outputCompressionQuality, guidanceScale, enhancePrompt,
-    // includeSafetyAttributes, includeRaiReason, outputGcsUri, labels,
-    // negativePrompt, language) are only valid on GenerateImagesConfig and
-    // would be rejected by the Gemini-native generateContent path. Pick only
-    // the fields that are valid on GenerateContentConfig instead of spreading
-    // the whole options object.
-    const nativeConfig: GenerateContentConfig = {}
-    if (modelOptions?.seed !== undefined) {
-      nativeConfig.seed = modelOptions.seed
+    // The portable `size` option is the baseline; modelOptions.imageConfig is
+    // the provider escape hatch and wins per field, so a caller passing only
+    // `imageConfig.imageSize` keeps the aspectRatio derived from `size`.
+    const imageConfig: ImageConfig = {
+      ...(parsedSize?.aspectRatio && { aspectRatio: parsedSize.aspectRatio }),
+      ...(parsedSize?.resolution && { imageSize: parsedSize.resolution }),
+      ...modelOptions?.imageConfig,
+    }
+
+    // Named picks, never a wholesale spread: the Imagen-shaped fields of
+    // GeminiImageProviderOptions (personGeneration, safetyFilterLevel,
+    // addWatermark, outputMimeType, …) are only valid on GenerateImagesConfig
+    // and would be rejected by generateContent. Picking by name means no
+    // Imagen field can reach this path even if one slips past the per-model
+    // provider-options map.
+    const nativeConfig: GenerateContentConfig = {
+      ...(modelOptions?.seed !== undefined && { seed: modelOptions.seed }),
+      ...(modelOptions?.safetySettings !== undefined && {
+        safetySettings: modelOptions.safetySettings,
+      }),
+      ...(modelOptions?.thinkingConfig !== undefined && {
+        thinkingConfig: modelOptions.thinkingConfig,
+      }),
+      ...(modelOptions?.systemInstruction !== undefined && {
+        systemInstruction: modelOptions.systemInstruction,
+      }),
     }
 
     const config: GenerateContentConfig = {
@@ -186,16 +200,7 @@ export class GeminiImageAdapter<
       // IMPORTANT: responseModalities is a protected default — set it AFTER
       // nativeConfig so nothing can silently disable image output.
       responseModalities: ['TEXT', 'IMAGE'],
-      ...(parsedSize && {
-        imageConfig: {
-          ...(parsedSize.aspectRatio && {
-            aspectRatio: parsedSize.aspectRatio,
-          }),
-          ...(parsedSize.resolution && {
-            imageSize: parsedSize.resolution,
-          }),
-        },
-      }),
+      ...(Object.keys(imageConfig).length > 0 && { imageConfig }),
     }
 
     const contents = this.buildContents(resolved, numberOfImages)
@@ -321,7 +326,7 @@ export class GeminiImageAdapter<
   }
 
   private buildImagenConfig(
-    options: ImageGenerationOptions<GeminiImageProviderOptions>,
+    options: ImageGenerationOptions<GeminiAnyImageProviderOptions>,
   ): GenerateImagesConfig {
     const { size, numberOfImages, modelOptions } = options
 
@@ -329,11 +334,62 @@ export class GeminiImageAdapter<
     // vendor `GenerateImagesConfig` fields are `field?: T` (no `| undefined`),
     // so we can only assign the property when we actually have a value.
     const sizeAspectRatio = size ? sizeToAspectRatio(size) : undefined
+
+    // Named picks, never a wholesale spread — the mirror image of the native
+    // path below. A native-only field (safetySettings, thinkingConfig,
+    // imageConfig, systemInstruction) belongs to GenerateContentConfig and is
+    // rejected by generateImages with 400 INVALID_ARGUMENT, so it must not be
+    // able to reach here even when the caller's `modelOptions` was typed
+    // against both shapes at once (e.g. an adapter inferred from a union of
+    // model names).
     return {
       numberOfImages: numberOfImages ?? 1,
-      // Map size to aspect ratio if provided (modelOptions.aspectRatio will override)
+      // Map size to aspect ratio if provided; modelOptions.aspectRatio,
+      // picked after it, overrides.
       ...(sizeAspectRatio !== undefined && { aspectRatio: sizeAspectRatio }),
-      ...modelOptions,
+      ...(modelOptions?.aspectRatio !== undefined && {
+        aspectRatio: modelOptions.aspectRatio,
+      }),
+      ...(modelOptions?.personGeneration !== undefined && {
+        personGeneration: modelOptions.personGeneration,
+      }),
+      ...(modelOptions?.safetyFilterLevel !== undefined && {
+        safetyFilterLevel: modelOptions.safetyFilterLevel,
+      }),
+      ...(modelOptions?.seed !== undefined && { seed: modelOptions.seed }),
+      ...(modelOptions?.addWatermark !== undefined && {
+        addWatermark: modelOptions.addWatermark,
+      }),
+      ...(modelOptions?.language !== undefined && {
+        language: modelOptions.language,
+      }),
+      ...(modelOptions?.negativePrompt !== undefined && {
+        negativePrompt: modelOptions.negativePrompt,
+      }),
+      ...(modelOptions?.outputMimeType !== undefined && {
+        outputMimeType: modelOptions.outputMimeType,
+      }),
+      ...(modelOptions?.outputCompressionQuality !== undefined && {
+        outputCompressionQuality: modelOptions.outputCompressionQuality,
+      }),
+      ...(modelOptions?.guidanceScale !== undefined && {
+        guidanceScale: modelOptions.guidanceScale,
+      }),
+      ...(modelOptions?.enhancePrompt !== undefined && {
+        enhancePrompt: modelOptions.enhancePrompt,
+      }),
+      ...(modelOptions?.includeSafetyAttributes !== undefined && {
+        includeSafetyAttributes: modelOptions.includeSafetyAttributes,
+      }),
+      ...(modelOptions?.includeRaiReason !== undefined && {
+        includeRaiReason: modelOptions.includeRaiReason,
+      }),
+      ...(modelOptions?.outputGcsUri !== undefined && {
+        outputGcsUri: modelOptions.outputGcsUri,
+      }),
+      ...(modelOptions?.labels !== undefined && {
+        labels: modelOptions.labels,
+      }),
     }
   }
 
@@ -392,6 +448,18 @@ export class GeminiImageAdapter<
   }
 }
 
+/** @deprecated Shut down 2026-06-25. Use `gemini-3.1-flash-image`. */
+export function createGeminiImage(
+  model: 'gemini-3.1-flash-image-preview',
+  apiKey: string,
+  config?: Omit<GeminiImageConfig, 'apiKey'>,
+): GeminiImageAdapter<'gemini-3.1-flash-image-preview'>
+/** @deprecated Shut down 2026-06-25. Use `gemini-3-pro-image`. */
+export function createGeminiImage(
+  model: 'gemini-3-pro-image-preview',
+  apiKey: string,
+  config?: Omit<GeminiImageConfig, 'apiKey'>,
+): GeminiImageAdapter<'gemini-3-pro-image-preview'>
 /**
  * Creates a Gemini image adapter with explicit API key.
  * Type resolution happens here at the call site.
@@ -411,6 +479,11 @@ export class GeminiImageAdapter<
  * });
  * ```
  */
+export function createGeminiImage<TModel extends GeminiImageModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<GeminiImageConfig, 'apiKey'>,
+): GeminiImageAdapter<TModel>
 export function createGeminiImage<TModel extends GeminiImageModel>(
   model: TModel,
   apiKey: string,
@@ -419,6 +492,16 @@ export function createGeminiImage<TModel extends GeminiImageModel>(
   return new GeminiImageAdapter({ apiKey, ...config }, model)
 }
 
+/** @deprecated Shut down 2026-06-25. Use `gemini-3.1-flash-image`. */
+export function geminiImage(
+  model: 'gemini-3.1-flash-image-preview',
+  config?: Omit<GeminiImageConfig, 'apiKey'>,
+): GeminiImageAdapter<'gemini-3.1-flash-image-preview'>
+/** @deprecated Shut down 2026-06-25. Use `gemini-3-pro-image`. */
+export function geminiImage(
+  model: 'gemini-3-pro-image-preview',
+  config?: Omit<GeminiImageConfig, 'apiKey'>,
+): GeminiImageAdapter<'gemini-3-pro-image-preview'>
 /**
  * Creates a Gemini image adapter with automatic API key detection from environment variables.
  * Type resolution happens here at the call site.
@@ -443,6 +526,10 @@ export function createGeminiImage<TModel extends GeminiImageModel>(
  * });
  * ```
  */
+export function geminiImage<TModel extends GeminiImageModel>(
+  model: TModel,
+  config?: Omit<GeminiImageConfig, 'apiKey'>,
+): GeminiImageAdapter<TModel>
 export function geminiImage<TModel extends GeminiImageModel>(
   model: TModel,
   config?: Omit<GeminiImageConfig, 'apiKey'>,