@tanstack/ai-gemini 0.20.0 → 0.21.0

package/dist/esm/realtime/utils.js

@@ -1,4 +1,8 @@
-const captureWorkletCode = `
+//#region src/realtime/utils.ts
+/**
+* Audio Worklet Processor for capturing and processing audio
+*/
+var captureWorkletCode = `
 class AudioCaptureProcessor extends AudioWorkletProcessor {
   constructor() {
     super();
@@ -38,7 +42,12 @@ class AudioCaptureProcessor extends AudioWorkletProcessor {
 
 // Register the processor
 registerProcessor("audio-capture-processor", AudioCaptureProcessor);`;
-const playbackWorkletCode = `
+/**
+* Audio Playback Worklet Processor for playing PCM audio.
+* Uses an offset tracker instead of slice() to avoid allocations
+* on the real-time audio thread.
+*/
+var playbackWorkletCode = `
 class PCMProcessor extends AudioWorkletProcessor {
   constructor() {
     super();
@@ -101,250 +110,198 @@ class PCMProcessor extends AudioWorkletProcessor {
 
 registerProcessor("pcm-processor", PCMProcessor);`;
 function calculateLevel(analyser) {
-  const data = new Uint8Array(analyser.fftSize);
-  analyser.getByteTimeDomainData(data);
-  let maxDeviation = 0;
-  for (const sample of data) {
-    const deviation = Math.abs(sample - 128);
-    if (deviation > maxDeviation) {
-      maxDeviation = deviation;
-    }
-  }
-  const normalized = maxDeviation / 128;
-  return Math.min(1, normalized * 1.5);
+	const data = new Uint8Array(analyser.fftSize);
+	analyser.getByteTimeDomainData(data);
+	let maxDeviation = 0;
+	for (const sample of data) {
+		const deviation = Math.abs(sample - 128);
+		if (deviation > maxDeviation) maxDeviation = deviation;
+	}
+	const normalized = maxDeviation / 128;
+	return Math.min(1, normalized * 1.5);
 }
 function base64ToArrayBuffer(base64) {
-  const binary = atob(base64);
-  const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0));
-  return bytes.buffer;
+	const binary = atob(base64);
+	return Uint8Array.from(binary, (char) => char.charCodeAt(0)).buffer;
 }
-const emptyFrequencyData = new Uint8Array(1024);
-const emptyTimeDomainData = new Uint8Array(2048).fill(128);
-class AudioStreamer {
-  audioContext = null;
-  audioWorklet = null;
-  mediaStream = null;
-  analyser = null;
-  isStreaming = false;
-  sampleRate = 16e3;
-  client = null;
-  constructor(client) {
-    this.client = client;
-  }
-  get inputLevel() {
-    if (!this.analyser) return 0;
-    return calculateLevel(this.analyser);
-  }
-  get inputFrequencyData() {
-    if (!this.analyser) return emptyFrequencyData;
-    const data = new Uint8Array(this.analyser.frequencyBinCount);
-    this.analyser.getByteFrequencyData(data);
-    return data;
-  }
-  get inputTimeDomainData() {
-    if (!this.analyser) return emptyTimeDomainData;
-    const data = new Uint8Array(this.analyser.fftSize);
-    this.analyser.getByteTimeDomainData(data);
-    return data;
-  }
-  get inputSampleRate() {
-    return this.sampleRate;
-  }
-  async start() {
-    try {
-      const audioConstraints = {
-        sampleRate: this.sampleRate,
-        echoCancellation: true,
-        noiseSuppression: true,
-        autoGainControl: true
-      };
-      this.mediaStream = await navigator.mediaDevices.getUserMedia({
-        audio: audioConstraints
-      });
-      const track = this.mediaStream.getAudioTracks()[0];
-      const settings = track?.getSettings();
-      if (settings?.autoGainControl) {
-        console.warn("Native AGC not supported.");
-      }
-      this.audioContext = new AudioContext({
-        sampleRate: this.sampleRate
-      });
-      if (this.audioContext.state === "suspended") {
-        await this.audioContext.resume();
-      }
-      const workletBlob = new Blob([captureWorkletCode], {
-        type: "application/javascript"
-      });
-      const workletUrl = URL.createObjectURL(workletBlob);
-      await this.audioContext.audioWorklet.addModule(workletUrl);
-      URL.revokeObjectURL(workletUrl);
-      this.audioWorklet = new AudioWorkletNode(
-        this.audioContext,
-        "audio-capture-processor"
-      );
-      this.audioWorklet.port.onmessage = (event) => {
-        if (!this.isStreaming) return;
-        if (event.data.type === "audio") {
-          const inputData = event.data.data;
-          const pcmData = this.convertToPCM16(inputData);
-          const base64Audio = this.arrayBufferToBase64(pcmData);
-          if (this.client?.isSetupComplete) {
-            this.client.sendAudioMessage(base64Audio);
-          }
-        }
-      };
-      this.analyser = this.audioContext.createAnalyser();
-      this.analyser.fftSize = 2048;
-      this.analyser.smoothingTimeConstant = 0.3;
-      const source = this.audioContext.createMediaStreamSource(this.mediaStream);
-      source.connect(this.analyser);
-      this.analyser.connect(this.audioWorklet);
-      this.isStreaming = true;
-    } catch (error) {
-      this.stop();
-      throw error;
-    }
-  }
-  stop() {
-    this.isStreaming = false;
-    if (this.audioWorklet) {
-      this.audioWorklet.disconnect();
-      this.audioWorklet.port.close();
-      this.audioWorklet = null;
-    }
-    if (this.audioContext) {
-      void this.audioContext.close();
-      this.audioContext = null;
-    }
-    if (this.mediaStream) {
-      this.mediaStream.getTracks().forEach((track) => track.stop());
-      this.mediaStream = null;
-    }
-  }
-  startAudioCapture() {
-    if (this.mediaStream) {
-      for (const track of this.mediaStream.getAudioTracks()) {
-        track.enabled = true;
-      }
-    }
-    this.isStreaming = true;
-  }
-  stopAudioCapture() {
-    if (this.mediaStream) {
-      for (const track of this.mediaStream.getAudioTracks()) {
-        track.enabled = false;
-      }
-    }
-    this.isStreaming = false;
-  }
-  convertToPCM16(float32Array) {
-    const int16Array = new Int16Array(float32Array.length);
-    for (let i = 0; i < float32Array.length; i++) {
-      const sample = Math.max(-1, Math.min(1, float32Array[i] ?? 0));
-      int16Array[i] = sample * 32767;
-    }
-    return int16Array.buffer;
-  }
-  arrayBufferToBase64(buffer) {
-    const bytes = new Uint8Array(buffer);
-    const binary = String.fromCharCode(...bytes);
-    return btoa(binary);
-  }
-}
-class AudioPlayer {
-  audioContext = null;
-  workletNode = null;
-  gainNode = null;
-  analyser = null;
-  isInitialized = false;
-  volume = 1;
-  sampleRate = 24e3;
-  get outputLevel() {
-    if (!this.analyser) return 0;
-    return calculateLevel(this.analyser);
-  }
-  get outputFrequencyData() {
-    if (!this.analyser) return emptyFrequencyData;
-    const data = new Uint8Array(this.analyser.frequencyBinCount);
-    this.analyser.getByteFrequencyData(data);
-    return data;
-  }
-  get outputTimeDomainData() {
-    if (!this.analyser) return emptyTimeDomainData;
-    const data = new Uint8Array(this.analyser.fftSize);
-    this.analyser.getByteTimeDomainData(data);
-    return data;
-  }
-  get outputSampleRate() {
-    return this.sampleRate;
-  }
-  async init() {
-    if (this.isInitialized) return;
-    try {
-      this.audioContext = new AudioContext({
-        sampleRate: this.sampleRate
-      });
-      const workletBlob = new Blob([playbackWorkletCode], {
-        type: "application/javascript"
-      });
-      const workletUrl = URL.createObjectURL(workletBlob);
-      await this.audioContext.audioWorklet.addModule(workletUrl);
-      URL.revokeObjectURL(workletUrl);
-      this.workletNode = new AudioWorkletNode(
-        this.audioContext,
-        "pcm-processor"
-      );
-      this.gainNode = this.audioContext.createGain();
-      this.gainNode.gain.value = this.volume;
-      this.analyser = this.audioContext.createAnalyser();
-      this.analyser.fftSize = 2048;
-      this.analyser.smoothingTimeConstant = 0.3;
-      this.workletNode.connect(this.gainNode);
-      this.gainNode.connect(this.analyser);
-      this.analyser.connect(this.audioContext.destination);
-      this.isInitialized = true;
-    } catch (error) {
-      this.destroy();
-      throw error;
-    }
-  }
-  async play(pcmData) {
-    if (!this.isInitialized) {
-      await this.init();
-    }
-    if (this.audioContext?.state === "suspended") {
-      await this.audioContext.resume();
-    }
-    const inputArray = new Int16Array(pcmData);
-    const float32Data = new Float32Array(inputArray.length);
-    for (let i = 0; i < inputArray.length; i++) {
-      float32Data[i] = (inputArray[i] ?? 0) / 32768;
-    }
-    this.workletNode?.port.postMessage(float32Data);
-  }
-  /* Interrupt playback */
-  interrupt() {
-    if (this.workletNode) {
-      this.workletNode.port.postMessage("interrupt");
-    }
-  }
-  setVolume(volume) {
-    this.volume = Math.max(0, Math.min(1, volume));
-    if (this.gainNode) {
-      this.gainNode.gain.value = this.volume;
-    }
-  }
-  destroy() {
-    if (this.audioContext) {
-      void this.audioContext.close();
-      this.audioContext = null;
-    }
-    this.isInitialized = false;
-  }
-}
-export {
-  AudioPlayer,
-  AudioStreamer,
-  base64ToArrayBuffer
+var emptyFrequencyData = /* @__PURE__ */ new Uint8Array(1024);
+var emptyTimeDomainData = (/* @__PURE__ */ new Uint8Array(2048)).fill(128);
+var AudioStreamer = class {
+	audioContext = null;
+	audioWorklet = null;
+	mediaStream = null;
+	analyser = null;
+	isStreaming = false;
+	sampleRate = 16e3;
+	client = null;
+	constructor(client) {
+		this.client = client;
+	}
+	get inputLevel() {
+		if (!this.analyser) return 0;
+		return calculateLevel(this.analyser);
+	}
+	get inputFrequencyData() {
+		if (!this.analyser) return emptyFrequencyData;
+		const data = new Uint8Array(this.analyser.frequencyBinCount);
+		this.analyser.getByteFrequencyData(data);
+		return data;
+	}
+	get inputTimeDomainData() {
+		if (!this.analyser) return emptyTimeDomainData;
+		const data = new Uint8Array(this.analyser.fftSize);
+		this.analyser.getByteTimeDomainData(data);
+		return data;
+	}
+	get inputSampleRate() {
+		return this.sampleRate;
+	}
+	async start() {
+		try {
+			const audioConstraints = {
+				sampleRate: this.sampleRate,
+				echoCancellation: true,
+				noiseSuppression: true,
+				autoGainControl: true
+			};
+			this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints });
+			if ((this.mediaStream.getAudioTracks()[0]?.getSettings())?.autoGainControl) console.warn("Native AGC not supported.");
+			this.audioContext = new AudioContext({ sampleRate: this.sampleRate });
+			if (this.audioContext.state === "suspended") await this.audioContext.resume();
+			const workletBlob = new Blob([captureWorkletCode], { type: "application/javascript" });
+			const workletUrl = URL.createObjectURL(workletBlob);
+			await this.audioContext.audioWorklet.addModule(workletUrl);
+			URL.revokeObjectURL(workletUrl);
+			this.audioWorklet = new AudioWorkletNode(this.audioContext, "audio-capture-processor");
+			this.audioWorklet.port.onmessage = (event) => {
+				if (!this.isStreaming) return;
+				if (event.data.type === "audio") {
+					const inputData = event.data.data;
+					const pcmData = this.convertToPCM16(inputData);
+					const base64Audio = this.arrayBufferToBase64(pcmData);
+					if (this.client?.isSetupComplete) this.client.sendAudioMessage(base64Audio);
+				}
+			};
+			this.analyser = this.audioContext.createAnalyser();
+			this.analyser.fftSize = 2048;
+			this.analyser.smoothingTimeConstant = .3;
+			this.audioContext.createMediaStreamSource(this.mediaStream).connect(this.analyser);
+			this.analyser.connect(this.audioWorklet);
+			this.isStreaming = true;
+		} catch (error) {
+			this.stop();
+			throw error;
+		}
+	}
+	stop() {
+		this.isStreaming = false;
+		if (this.audioWorklet) {
+			this.audioWorklet.disconnect();
+			this.audioWorklet.port.close();
+			this.audioWorklet = null;
+		}
+		if (this.audioContext) {
+			this.audioContext.close();
+			this.audioContext = null;
+		}
+		if (this.mediaStream) {
+			this.mediaStream.getTracks().forEach((track) => track.stop());
+			this.mediaStream = null;
+		}
+	}
+	startAudioCapture() {
+		if (this.mediaStream) for (const track of this.mediaStream.getAudioTracks()) track.enabled = true;
+		this.isStreaming = true;
+	}
+	stopAudioCapture() {
+		if (this.mediaStream) for (const track of this.mediaStream.getAudioTracks()) track.enabled = false;
+		this.isStreaming = false;
+	}
+	convertToPCM16(float32Array) {
+		const int16Array = new Int16Array(float32Array.length);
+		for (let i = 0; i < float32Array.length; i++) int16Array[i] = Math.max(-1, Math.min(1, float32Array[i] ?? 0)) * 32767;
+		return int16Array.buffer;
+	}
+	arrayBufferToBase64(buffer) {
+		const bytes = new Uint8Array(buffer);
+		const binary = String.fromCharCode(...bytes);
+		return btoa(binary);
+	}
 };
-//# sourceMappingURL=utils.js.map
+var AudioPlayer = class {
+	audioContext = null;
+	workletNode = null;
+	gainNode = null;
+	analyser = null;
+	isInitialized = false;
+	volume = 1;
+	sampleRate = 24e3;
+	get outputLevel() {
+		if (!this.analyser) return 0;
+		return calculateLevel(this.analyser);
+	}
+	get outputFrequencyData() {
+		if (!this.analyser) return emptyFrequencyData;
+		const data = new Uint8Array(this.analyser.frequencyBinCount);
+		this.analyser.getByteFrequencyData(data);
+		return data;
+	}
+	get outputTimeDomainData() {
+		if (!this.analyser) return emptyTimeDomainData;
+		const data = new Uint8Array(this.analyser.fftSize);
+		this.analyser.getByteTimeDomainData(data);
+		return data;
+	}
+	get outputSampleRate() {
+		return this.sampleRate;
+	}
+	async init() {
+		if (this.isInitialized) return;
+		try {
+			this.audioContext = new AudioContext({ sampleRate: this.sampleRate });
+			const workletBlob = new Blob([playbackWorkletCode], { type: "application/javascript" });
+			const workletUrl = URL.createObjectURL(workletBlob);
+			await this.audioContext.audioWorklet.addModule(workletUrl);
+			URL.revokeObjectURL(workletUrl);
+			this.workletNode = new AudioWorkletNode(this.audioContext, "pcm-processor");
+			this.gainNode = this.audioContext.createGain();
+			this.gainNode.gain.value = this.volume;
+			this.analyser = this.audioContext.createAnalyser();
+			this.analyser.fftSize = 2048;
+			this.analyser.smoothingTimeConstant = .3;
+			this.workletNode.connect(this.gainNode);
+			this.gainNode.connect(this.analyser);
+			this.analyser.connect(this.audioContext.destination);
+			this.isInitialized = true;
+		} catch (error) {
+			this.destroy();
+			throw error;
+		}
+	}
+	async play(pcmData) {
+		if (!this.isInitialized) await this.init();
+		if (this.audioContext?.state === "suspended") await this.audioContext.resume();
+		const inputArray = new Int16Array(pcmData);
+		const float32Data = new Float32Array(inputArray.length);
+		for (let i = 0; i < inputArray.length; i++) float32Data[i] = (inputArray[i] ?? 0) / 32768;
+		this.workletNode?.port.postMessage(float32Data);
+	}
+	interrupt() {
+		if (this.workletNode) this.workletNode.port.postMessage("interrupt");
+	}
+	setVolume(volume) {
+		this.volume = Math.max(0, Math.min(1, volume));
+		if (this.gainNode) this.gainNode.gain.value = this.volume;
+	}
+	destroy() {
+		if (this.audioContext) {
+			this.audioContext.close();
+			this.audioContext = null;
+		}
+		this.isInitialized = false;
+	}
+};
+//#endregion
+export { AudioPlayer, AudioStreamer, base64ToArrayBuffer };
+
+//# sourceMappingURL=utils.js.map

package/dist/esm/realtime/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sources":["../../../src/realtime/utils.ts"],"sourcesContent":["import type { GeminiLiveClient } from './client'\n\n/**\n * Audio Worklet Processor for capturing and processing audio\n */\nconst captureWorkletCode = `\nclass AudioCaptureProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.bufferSize = 512; // 32ms at 16kHz — per Gemini best practices (20-40ms chunks)\n    this.buffer = new Float32Array(this.bufferSize);\n    this.bufferIndex = 0;\n  }\n\n  process(inputs, outputs, parameters) {\n    const input = inputs[0];\n\n    if (input && input.length > 0) {\n      const inputChannel = input[0];\n\n      // Buffer the incoming audio\n      for (let i = 0; i < inputChannel.length; i++) {\n        this.buffer[this.bufferIndex++] = inputChannel[i];\n\n        // When buffer is full, send it to main thread\n        if (this.bufferIndex >= this.bufferSize) {\n          // Send the buffered audio to the main thread\n          this.port.postMessage({\n            type: \"audio\",\n            data: this.buffer.slice(),\n          });\n\n          // Reset buffer\n          this.bufferIndex = 0;\n        }\n      }\n    }\n\n    // Return true to keep the processor alive\n    return true;\n  }\n}\n\n// Register the processor\nregisterProcessor(\"audio-capture-processor\", AudioCaptureProcessor);`\n\n/**\n * Audio Playback Worklet Processor for playing PCM audio.\n * Uses an offset tracker instead of slice() to avoid allocations\n * on the real-time audio thread.\n */\nconst playbackWorkletCode = `\nclass PCMProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.audioQueue = [];\n    this.currentOffset = 0; // Track position in current buffer (avoids slice())\n\n    this.port.onmessage = (event) => {\n      if (event.data === \"interrupt\") {\n        // Clear the queue on interrupt\n        this.audioQueue = [];\n        this.currentOffset = 0;\n      } else if (event.data instanceof Float32Array) {\n        // Add audio data to the queue\n        this.audioQueue.push(event.data);\n      }\n    };\n  }\n\n  process(inputs, outputs, parameters) {\n    const output = outputs[0];\n    if (output.length === 0) return true;\n\n    const channel = output[0];\n    let outputIndex = 0;\n\n    // Fill the output buffer from the queue\n    while (outputIndex < channel.length && this.audioQueue.length > 0) {\n      const currentBuffer = this.audioQueue[0];\n\n      if (!currentBuffer || currentBuffer.length === 0) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n        continue;\n      }\n\n      const remainingOutput = channel.length - outputIndex;\n      const remainingBuffer = currentBuffer.length - this.currentOffset;\n      const copyLength = Math.min(remainingOutput, remainingBuffer);\n\n      // Copy audio data to output using offset (no slice allocation)\n      for (let i = 0; i < copyLength; i++) {\n        channel[outputIndex++] = currentBuffer[this.currentOffset++];\n      }\n\n      // If we've consumed the entire buffer, move to the next one\n      if (this.currentOffset >= currentBuffer.length) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n      }\n    }\n\n    // Fill remaining output with silence\n    while (outputIndex < channel.length) {\n      channel[outputIndex++] = 0;\n    }\n\n    return true;\n  }\n}\n\nregisterProcessor(\"pcm-processor\", PCMProcessor);`\n\nfunction calculateLevel(analyser: AnalyserNode): number {\n  const data = new Uint8Array(analyser.fftSize)\n  analyser.getByteTimeDomainData(data)\n\n  // Find peak deviation from center (128 is silence)\n  // This is more responsive than RMS for voice level meters\n  let maxDeviation = 0\n  for (const sample of data) {\n    const deviation = Math.abs(sample - 128)\n    if (deviation > maxDeviation) {\n      maxDeviation = deviation\n    }\n  }\n\n  // Normalize to 0-1 range (max deviation is 128)\n  // Scale by 1.5x so that ~66% amplitude reads as full scale\n  // This provides good visual feedback without pegging too early\n  const normalized = maxDeviation / 128\n  return Math.min(1, normalized * 1.5)\n}\n\nexport function base64ToArrayBuffer(base64: string): ArrayBuffer {\n  const binary = atob(base64)\n  const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0))\n  return bytes.buffer\n}\n\n// Empty arrays for when visualization isn't available\n// frequencyBinCount = fftSize / 2 = 1024\nconst emptyFrequencyData = new Uint8Array(1024)\nconst emptyTimeDomainData = new Uint8Array(2048).fill(128) // 128 is silence\n\nexport class AudioStreamer {\n  private audioContext: AudioContext | null = null\n  private audioWorklet: AudioWorkletNode | null = null\n  private mediaStream: MediaStream | null = null\n  private analyser: AnalyserNode | null = null\n  private isStreaming = false\n  private readonly sampleRate = 16000\n  private readonly client: GeminiLiveClient | null = null\n\n  constructor(client: GeminiLiveClient) {\n    this.client = client\n  }\n\n  get inputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get inputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get inputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get inputSampleRate() {\n    return this.sampleRate\n  }\n\n  async start() {\n    try {\n      const audioConstraints: MediaTrackConstraints = {\n        sampleRate: this.sampleRate,\n        echoCancellation: true,\n        noiseSuppression: true,\n        autoGainControl: true,\n      }\n\n      // Get microphone access\n      this.mediaStream = await navigator.mediaDevices.getUserMedia({\n        audio: audioConstraints,\n      })\n\n      // Check if native AGC is active\n      const track = this.mediaStream.getAudioTracks()[0]\n      const settings = track?.getSettings()\n\n      if (settings?.autoGainControl) {\n        console.warn('Native AGC not supported.')\n      }\n\n      // Create audio context\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      if (this.audioContext.state === 'suspended') {\n        await this.audioContext.resume()\n      }\n\n      const workletBlob = new Blob([captureWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create the audio worklet node\n      this.audioWorklet = new AudioWorkletNode(\n        this.audioContext,\n        'audio-capture-processor',\n      )\n\n      // Set up message handling from the worklet\n      this.audioWorklet.port.onmessage = (event) => {\n        if (!this.isStreaming) return\n\n        if (event.data.type === 'audio') {\n          const inputData = event.data.data\n          const pcmData = this.convertToPCM16(inputData)\n          const base64Audio = this.arrayBufferToBase64(pcmData)\n\n          // Send to Gemini only if after setup complete\n          if (this.client?.isSetupComplete) {\n            this.client.sendAudioMessage(base64Audio)\n          }\n        }\n      }\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect the audio graph\n      const source = this.audioContext.createMediaStreamSource(this.mediaStream)\n      source.connect(this.analyser)\n      this.analyser.connect(this.audioWorklet)\n\n      // Start streaming\n      this.isStreaming = true\n    } catch (error) {\n      // Clean up the mic + audio context if setup failed partway through.\n      this.stop()\n      throw error\n    }\n  }\n\n  stop() {\n    this.isStreaming = false\n\n    if (this.audioWorklet) {\n      this.audioWorklet.disconnect()\n      this.audioWorklet.port.close()\n      this.audioWorklet = null\n    }\n\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n\n    if (this.mediaStream) {\n      this.mediaStream.getTracks().forEach((track) => track.stop())\n      this.mediaStream = null\n    }\n  }\n\n  startAudioCapture() {\n    if (this.mediaStream) {\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = true\n      }\n    }\n    this.isStreaming = true\n  }\n\n  stopAudioCapture() {\n    if (this.mediaStream) {\n      // Disable tracks rather than stopping them to allow re-enabling\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = false\n      }\n    }\n    this.isStreaming = false\n  }\n\n  private convertToPCM16(float32Array: Float32Array): ArrayBuffer {\n    const int16Array = new Int16Array(float32Array.length)\n    for (let i = 0; i < float32Array.length; i++) {\n      const sample = Math.max(-1, Math.min(1, float32Array[i] ?? 0))\n      int16Array[i] = sample * 0x7fff\n    }\n    return int16Array.buffer\n  }\n\n  private arrayBufferToBase64(buffer: ArrayBuffer): string {\n    const bytes = new Uint8Array(buffer)\n    const binary = String.fromCharCode(...bytes)\n    return btoa(binary)\n  }\n}\n\nexport class AudioPlayer {\n  private audioContext: AudioContext | null = null\n  private workletNode: AudioWorkletNode | null = null\n  private gainNode: GainNode | null = null\n  private analyser: AnalyserNode | null = null\n  private isInitialized = false\n  private volume = 1.0\n  private readonly sampleRate = 24000\n\n  get outputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get outputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get outputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get outputSampleRate() {\n    return this.sampleRate\n  }\n\n  async init() {\n    if (this.isInitialized) return\n\n    try {\n      // Create audio context at 24kHz to match Gemini\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      const workletBlob = new Blob([playbackWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create worklet node\n      this.workletNode = new AudioWorkletNode(\n        this.audioContext,\n        'pcm-processor',\n      )\n\n      // Create gain node for volume control\n      this.gainNode = this.audioContext.createGain()\n      this.gainNode.gain.value = this.volume\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect nodes\n      this.workletNode.connect(this.gainNode)\n      this.gainNode.connect(this.analyser)\n      this.analyser.connect(this.audioContext.destination)\n\n      this.isInitialized = true\n    } catch (error) {\n      // Release the audio context if initialization failed partway through.\n      this.destroy()\n      throw error\n    }\n  }\n\n  async play(pcmData: ArrayBuffer) {\n    if (!this.isInitialized) {\n      await this.init()\n    }\n\n    // Resume audio context if suspended\n    if (this.audioContext?.state === 'suspended') {\n      await this.audioContext.resume()\n    }\n\n    // Convert PCM16 LE to Float32\n    const inputArray = new Int16Array(pcmData)\n    const float32Data = new Float32Array(inputArray.length)\n    for (let i = 0; i < inputArray.length; i++) {\n      float32Data[i] = (inputArray[i] ?? 0) / 32768\n    }\n\n    // Send to worklet for playback\n    this.workletNode?.port.postMessage(float32Data)\n  }\n\n  /* Interrupt playback */\n  interrupt() {\n    if (this.workletNode) {\n      this.workletNode.port.postMessage('interrupt')\n    }\n  }\n\n  setVolume(volume: number) {\n    this.volume = Math.max(0, Math.min(1, volume))\n    if (this.gainNode) {\n      this.gainNode.gain.value = this.volume\n    }\n  }\n\n  destroy() {\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n    this.isInitialized = false\n  }\n}\n"],"names":[],"mappings":"AAKA,MAAM,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA8C3B,MAAM,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+D5B,SAAS,eAAe,UAAgC;AACtD,QAAM,OAAO,IAAI,WAAW,SAAS,OAAO;AAC5C,WAAS,sBAAsB,IAAI;AAInC,MAAI,eAAe;AACnB,aAAW,UAAU,MAAM;AACzB,UAAM,YAAY,KAAK,IAAI,SAAS,GAAG;AACvC,QAAI,YAAY,cAAc;AAC5B,qBAAe;AAAA,IACjB;AAAA,EACF;AAKA,QAAM,aAAa,eAAe;AAClC,SAAO,KAAK,IAAI,GAAG,aAAa,GAAG;AACrC;AAEO,SAAS,oBAAoB,QAA6B;AAC/D,QAAM,SAAS,KAAK,MAAM;AAC1B,QAAM,QAAQ,WAAW,KAAK,QAAQ,CAAC,SAAS,KAAK,WAAW,CAAC,CAAC;AAClE,SAAO,MAAM;AACf;AAIA,MAAM,qBAAqB,IAAI,WAAW,IAAI;AAC9C,MAAM,sBAAsB,IAAI,WAAW,IAAI,EAAE,KAAK,GAAG;AAElD,MAAM,cAAc;AAAA,EACjB,eAAoC;AAAA,EACpC,eAAwC;AAAA,EACxC,cAAkC;AAAA,EAClC,WAAgC;AAAA,EAChC,cAAc;AAAA,EACL,aAAa;AAAA,EACb,SAAkC;AAAA,EAEnD,YAAY,QAA0B;AACpC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAI,aAAa;AACf,QAAI,CAAC,KAAK,SAAU,QAAO;AAC3B,WAAO,eAAe,KAAK,QAAQ;AAAA,EACrC;AAAA,EAEA,IAAI,qBAAqB;AACvB,QAAI,CAAC,KAAK,SAAU,QAAO;AAC3B,UAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;AAC3D,SAAK,SAAS,qBAAqB,IAAI;AACvC,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,sBAAsB;AACxB,QAAI,CAAC,KAAK,SAAU,QAAO;AAC3B,UAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;AACjD,SAAK,SAAS,sBAAsB,IAAI;AACxC,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,kBAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI;AACF,YAAM,mBAA0C;AAAA,QAC9C,YAAY,KAAK;AAAA,QACjB,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,QAClB,iBAAiB;AAAA,MAAA;AAInB,WAAK,cAAc,MAAM,UAAU,aAAa,aAAa;AAAA,QAC3D,OAAO;AAAA,MAAA,CACR;AAGD,YAAM,QAAQ,KAAK,YAAY,eAAA,EAAiB,CAAC;AACjD,YAAM,WAAW,OAAO,YAAA;AAExB,UAAI,UAAU,iBAAiB;AAC7B,gBAAQ,KAAK,2BAA2B;AAAA,MAC1C;AAGA,WAAK,eAAe,IAAI,aAAa;AAAA,QACnC,YAAY,KAAK;AAAA,MAAA,CAClB;AAED,UAAI,KAAK,aAAa,UAAU,aAAa;AAC3C,cAAM,KAAK,aAAa,OAAA;AAAA,MAC1B;AAEA,YAAM,cAAc,IAAI,KAAK,CAAC,kBAAkB,GAAG;AAAA,QACjD,MAAM;AAAA,MAAA,CACP;AACD,YAAM,aAAa,IAAI,gBAAgB,WAAW;AAGlD,YAAM,KAAK,aAAa,aAAa,UAAU,UAAU;AACzD,UAAI,gBAAgB,UAAU;AAG9B,WAAK,eAAe,IAAI;AAAA,QACtB,KAAK;AAAA,QACL;AAAA,MAAA;AAIF,WAAK,aAAa,KAAK,YAAY,CAAC,UAAU;AAC5C,YAAI,CAAC,KAAK,YAAa;AAEvB,YAAI,MAAM,KAAK,SAAS,SAAS;AAC/B,gBAAM,YAAY,MAAM,KAAK;AAC7B,gBAAM,UAAU,KAAK,eAAe,SAAS;AAC7C,gBAAM,cAAc,KAAK,oBAAoB,OAAO;AAGpD,cAAI,KAAK,QAAQ,iBAAiB;AAChC,iBAAK,OAAO,iBAAiB,WAAW;AAAA,UAC1C;AAAA,QACF;AAAA,MACF;AAGA,WAAK,WAAW,KAAK,aAAa,eAAA;AAClC,WAAK,SAAS,UAAU;AACxB,WAAK,SAAS,wBAAwB;AAGtC,YAAM,SAAS,KAAK,aAAa,wBAAwB,KAAK,WAAW;AACzE,aAAO,QAAQ,KAAK,QAAQ;AAC5B,WAAK,SAAS,QAAQ,KAAK,YAAY;AAGvC,WAAK,cAAc;AAAA,IACrB,SAAS,OAAO;AAEd,WAAK,KAAA;AACL,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,OAAO;AACL,SAAK,cAAc;AAEnB,QAAI,KAAK,cAAc;AACrB,WAAK,aAAa,WAAA;AAClB,WAAK,aAAa,KAAK,MAAA;AACvB,WAAK,eAAe;AAAA,IACtB;AAEA,QAAI,KAAK,cAAc;AACrB,WAAK,KAAK,aAAa,MAAA;AACvB,WAAK,eAAe;AAAA,IACtB;AAEA,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,YAAY,QAAQ,CAAC,UAAU,MAAM,MAAM;AAC5D,WAAK,cAAc;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,oBAAoB;AAClB,QAAI,KAAK,aAAa;AACpB,iBAAW,SAAS,KAAK,YAAY,eAAA,GAAkB;AACrD,cAAM,UAAU;AAAA,MAClB;AAAA,IACF;AACA,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,mBAAmB;AACjB,QAAI,KAAK,aAAa;AAEpB,iBAAW,SAAS,KAAK,YAAY,eAAA,GAAkB;AACrD,cAAM,UAAU;AAAA,MAClB;AAAA,IACF;AACA,SAAK,cAAc;AAAA,EACrB;AAAA,EAEQ,eAAe,cAAyC;AAC9D,UAAM,aAAa,IAAI,WAAW,aAAa,MAAM;AACrD,aAAS,IAAI,GAAG,IAAI,aAAa,QAAQ,KAAK;AAC5C,YAAM,SAAS,KAAK,IAAI,IAAI,KAAK,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;AAC7D,iBAAW,CAAC,IAAI,SAAS;AAAA,IAC3B;AACA,WAAO,WAAW;AAAA,EACpB;AAAA,EAEQ,oBAAoB,QAA6B;AACvD,UAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,UAAM,SAAS,OAAO,aAAa,GAAG,KAAK;AAC3C,WAAO,KAAK,MAAM;AAAA,EACpB;AACF;AAEO,MAAM,YAAY;AAAA,EACf,eAAoC;AAAA,EACpC,cAAuC;AAAA,EACvC,WAA4B;AAAA,EAC5B,WAAgC;AAAA,EAChC,gBAAgB;AAAA,EAChB,SAAS;AAAA,EACA,aAAa;AAAA,EAE9B,IAAI,cAAc;AAChB,QAAI,CAAC,KAAK,SAAU,QAAO;AAC3B,WAAO,eAAe,KAAK,QAAQ;AAAA,EACrC;AAAA,EAEA,IAAI,sBAAsB;AACxB,QAAI,CAAC,KAAK,SAAU,QAAO;AAC3B,UAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;AAC3D,SAAK,SAAS,qBAAqB,IAAI;AACvC,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,uBAAuB;AACzB,QAAI,CAAC,KAAK,SAAU,QAAO;AAC3B,UAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;AACjD,SAAK,SAAS,sBAAsB,IAAI;AACxC,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,mBAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,KAAK,cAAe;AAExB,QAAI;AAEF,WAAK,eAAe,IAAI,aAAa;AAAA,QACnC,YAAY,KAAK;AAAA,MAAA,CAClB;AAED,YAAM,cAAc,IAAI,KAAK,CAAC,mBAAmB,GAAG;AAAA,QAClD,MAAM;AAAA,MAAA,CACP;AACD,YAAM,aAAa,IAAI,gBAAgB,WAAW;AAGlD,YAAM,KAAK,aAAa,aAAa,UAAU,UAAU;AACzD,UAAI,gBAAgB,UAAU;AAG9B,WAAK,cAAc,IAAI;AAAA,QACrB,KAAK;AAAA,QACL;AAAA,MAAA;AAIF,WAAK,WAAW,KAAK,aAAa,WAAA;AAClC,WAAK,SAAS,KAAK,QAAQ,KAAK;AAGhC,WAAK,WAAW,KAAK,aAAa,eAAA;AAClC,WAAK,SAAS,UAAU;AACxB,WAAK,SAAS,wBAAwB;AAGtC,WAAK,YAAY,QAAQ,KAAK,QAAQ;AACtC,WAAK,SAAS,QAAQ,KAAK,QAAQ;AACnC,WAAK,SAAS,QAAQ,KAAK,aAAa,WAAW;AAEnD,WAAK,gBAAgB;AAAA,IACvB,SAAS,OAAO;AAEd,WAAK,QAAA;AACL,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,SAAsB;AAC/B,QAAI,CAAC,KAAK,eAAe;AACvB,YAAM,KAAK,KAAA;AAAA,IACb;AAGA,QAAI,KAAK,cAAc,UAAU,aAAa;AAC5C,YAAM,KAAK,aAAa,OAAA;AAAA,IAC1B;AAGA,UAAM,aAAa,IAAI,WAAW,OAAO;AACzC,UAAM,cAAc,IAAI,aAAa,WAAW,MAAM;AACtD,aAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,kBAAY,CAAC,KAAK,WAAW,CAAC,KAAK,KAAK;AAAA,IAC1C;AAGA,SAAK,aAAa,KAAK,YAAY,WAAW;AAAA,EAChD;AAAA;AAAA,EAGA,YAAY;AACV,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,KAAK,YAAY,WAAW;AAAA,IAC/C;AAAA,EACF;AAAA,EAEA,UAAU,QAAgB;AACxB,SAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;AAC7C,QAAI,KAAK,UAAU;AACjB,WAAK,SAAS,KAAK,QAAQ,KAAK;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,UAAU;AACR,QAAI,KAAK,cAAc;AACrB,WAAK,KAAK,aAAa,MAAA;AACvB,WAAK,eAAe;AAAA,IACtB;AACA,SAAK,gBAAgB;AAAA,EACvB;AACF;"}
+{"version":3,"file":"utils.js","names":[],"sources":["../../../src/realtime/utils.ts"],"sourcesContent":["import type { GeminiLiveClient } from './client'\n\n/**\n * Audio Worklet Processor for capturing and processing audio\n */\nconst captureWorkletCode = `\nclass AudioCaptureProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.bufferSize = 512; // 32ms at 16kHz — per Gemini best practices (20-40ms chunks)\n    this.buffer = new Float32Array(this.bufferSize);\n    this.bufferIndex = 0;\n  }\n\n  process(inputs, outputs, parameters) {\n    const input = inputs[0];\n\n    if (input && input.length > 0) {\n      const inputChannel = input[0];\n\n      // Buffer the incoming audio\n      for (let i = 0; i < inputChannel.length; i++) {\n        this.buffer[this.bufferIndex++] = inputChannel[i];\n\n        // When buffer is full, send it to main thread\n        if (this.bufferIndex >= this.bufferSize) {\n          // Send the buffered audio to the main thread\n          this.port.postMessage({\n            type: \"audio\",\n            data: this.buffer.slice(),\n          });\n\n          // Reset buffer\n          this.bufferIndex = 0;\n        }\n      }\n    }\n\n    // Return true to keep the processor alive\n    return true;\n  }\n}\n\n// Register the processor\nregisterProcessor(\"audio-capture-processor\", AudioCaptureProcessor);`\n\n/**\n * Audio Playback Worklet Processor for playing PCM audio.\n * Uses an offset tracker instead of slice() to avoid allocations\n * on the real-time audio thread.\n */\nconst playbackWorkletCode = `\nclass PCMProcessor extends AudioWorkletProcessor {\n  constructor() {\n    super();\n    this.audioQueue = [];\n    this.currentOffset = 0; // Track position in current buffer (avoids slice())\n\n    this.port.onmessage = (event) => {\n      if (event.data === \"interrupt\") {\n        // Clear the queue on interrupt\n        this.audioQueue = [];\n        this.currentOffset = 0;\n      } else if (event.data instanceof Float32Array) {\n        // Add audio data to the queue\n        this.audioQueue.push(event.data);\n      }\n    };\n  }\n\n  process(inputs, outputs, parameters) {\n    const output = outputs[0];\n    if (output.length === 0) return true;\n\n    const channel = output[0];\n    let outputIndex = 0;\n\n    // Fill the output buffer from the queue\n    while (outputIndex < channel.length && this.audioQueue.length > 0) {\n      const currentBuffer = this.audioQueue[0];\n\n      if (!currentBuffer || currentBuffer.length === 0) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n        continue;\n      }\n\n      const remainingOutput = channel.length - outputIndex;\n      const remainingBuffer = currentBuffer.length - this.currentOffset;\n      const copyLength = Math.min(remainingOutput, remainingBuffer);\n\n      // Copy audio data to output using offset (no slice allocation)\n      for (let i = 0; i < copyLength; i++) {\n        channel[outputIndex++] = currentBuffer[this.currentOffset++];\n      }\n\n      // If we've consumed the entire buffer, move to the next one\n      if (this.currentOffset >= currentBuffer.length) {\n        this.audioQueue.shift();\n        this.currentOffset = 0;\n      }\n    }\n\n    // Fill remaining output with silence\n    while (outputIndex < channel.length) {\n      channel[outputIndex++] = 0;\n    }\n\n    return true;\n  }\n}\n\nregisterProcessor(\"pcm-processor\", PCMProcessor);`\n\nfunction calculateLevel(analyser: AnalyserNode): number {\n  const data = new Uint8Array(analyser.fftSize)\n  analyser.getByteTimeDomainData(data)\n\n  // Find peak deviation from center (128 is silence)\n  // This is more responsive than RMS for voice level meters\n  let maxDeviation = 0\n  for (const sample of data) {\n    const deviation = Math.abs(sample - 128)\n    if (deviation > maxDeviation) {\n      maxDeviation = deviation\n    }\n  }\n\n  // Normalize to 0-1 range (max deviation is 128)\n  // Scale by 1.5x so that ~66% amplitude reads as full scale\n  // This provides good visual feedback without pegging too early\n  const normalized = maxDeviation / 128\n  return Math.min(1, normalized * 1.5)\n}\n\nexport function base64ToArrayBuffer(base64: string): ArrayBuffer {\n  const binary = atob(base64)\n  const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0))\n  return bytes.buffer\n}\n\n// Empty arrays for when visualization isn't available\n// frequencyBinCount = fftSize / 2 = 1024\nconst emptyFrequencyData = new Uint8Array(1024)\nconst emptyTimeDomainData = new Uint8Array(2048).fill(128) // 128 is silence\n\nexport class AudioStreamer {\n  private audioContext: AudioContext | null = null\n  private audioWorklet: AudioWorkletNode | null = null\n  private mediaStream: MediaStream | null = null\n  private analyser: AnalyserNode | null = null\n  private isStreaming = false\n  private readonly sampleRate = 16000\n  private readonly client: GeminiLiveClient | null = null\n\n  constructor(client: GeminiLiveClient) {\n    this.client = client\n  }\n\n  get inputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get inputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get inputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get inputSampleRate() {\n    return this.sampleRate\n  }\n\n  async start() {\n    try {\n      const audioConstraints: MediaTrackConstraints = {\n        sampleRate: this.sampleRate,\n        echoCancellation: true,\n        noiseSuppression: true,\n        autoGainControl: true,\n      }\n\n      // Get microphone access\n      this.mediaStream = await navigator.mediaDevices.getUserMedia({\n        audio: audioConstraints,\n      })\n\n      // Check if native AGC is active\n      const track = this.mediaStream.getAudioTracks()[0]\n      const settings = track?.getSettings()\n\n      if (settings?.autoGainControl) {\n        console.warn('Native AGC not supported.')\n      }\n\n      // Create audio context\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      if (this.audioContext.state === 'suspended') {\n        await this.audioContext.resume()\n      }\n\n      const workletBlob = new Blob([captureWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create the audio worklet node\n      this.audioWorklet = new AudioWorkletNode(\n        this.audioContext,\n        'audio-capture-processor',\n      )\n\n      // Set up message handling from the worklet\n      this.audioWorklet.port.onmessage = (event) => {\n        if (!this.isStreaming) return\n\n        if (event.data.type === 'audio') {\n          const inputData = event.data.data\n          const pcmData = this.convertToPCM16(inputData)\n          const base64Audio = this.arrayBufferToBase64(pcmData)\n\n          // Send to Gemini only if after setup complete\n          if (this.client?.isSetupComplete) {\n            this.client.sendAudioMessage(base64Audio)\n          }\n        }\n      }\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect the audio graph\n      const source = this.audioContext.createMediaStreamSource(this.mediaStream)\n      source.connect(this.analyser)\n      this.analyser.connect(this.audioWorklet)\n\n      // Start streaming\n      this.isStreaming = true\n    } catch (error) {\n      // Clean up the mic + audio context if setup failed partway through.\n      this.stop()\n      throw error\n    }\n  }\n\n  stop() {\n    this.isStreaming = false\n\n    if (this.audioWorklet) {\n      this.audioWorklet.disconnect()\n      this.audioWorklet.port.close()\n      this.audioWorklet = null\n    }\n\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n\n    if (this.mediaStream) {\n      this.mediaStream.getTracks().forEach((track) => track.stop())\n      this.mediaStream = null\n    }\n  }\n\n  startAudioCapture() {\n    if (this.mediaStream) {\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = true\n      }\n    }\n    this.isStreaming = true\n  }\n\n  stopAudioCapture() {\n    if (this.mediaStream) {\n      // Disable tracks rather than stopping them to allow re-enabling\n      for (const track of this.mediaStream.getAudioTracks()) {\n        track.enabled = false\n      }\n    }\n    this.isStreaming = false\n  }\n\n  private convertToPCM16(float32Array: Float32Array): ArrayBuffer {\n    const int16Array = new Int16Array(float32Array.length)\n    for (let i = 0; i < float32Array.length; i++) {\n      const sample = Math.max(-1, Math.min(1, float32Array[i] ?? 0))\n      int16Array[i] = sample * 0x7fff\n    }\n    return int16Array.buffer\n  }\n\n  private arrayBufferToBase64(buffer: ArrayBuffer): string {\n    const bytes = new Uint8Array(buffer)\n    const binary = String.fromCharCode(...bytes)\n    return btoa(binary)\n  }\n}\n\nexport class AudioPlayer {\n  private audioContext: AudioContext | null = null\n  private workletNode: AudioWorkletNode | null = null\n  private gainNode: GainNode | null = null\n  private analyser: AnalyserNode | null = null\n  private isInitialized = false\n  private volume = 1.0\n  private readonly sampleRate = 24000\n\n  get outputLevel() {\n    if (!this.analyser) return 0\n    return calculateLevel(this.analyser)\n  }\n\n  get outputFrequencyData() {\n    if (!this.analyser) return emptyFrequencyData\n    const data = new Uint8Array(this.analyser.frequencyBinCount)\n    this.analyser.getByteFrequencyData(data)\n    return data\n  }\n\n  get outputTimeDomainData() {\n    if (!this.analyser) return emptyTimeDomainData\n    const data = new Uint8Array(this.analyser.fftSize)\n    this.analyser.getByteTimeDomainData(data)\n    return data\n  }\n\n  get outputSampleRate() {\n    return this.sampleRate\n  }\n\n  async init() {\n    if (this.isInitialized) return\n\n    try {\n      // Create audio context at 24kHz to match Gemini\n      this.audioContext = new AudioContext({\n        sampleRate: this.sampleRate,\n      })\n\n      const workletBlob = new Blob([playbackWorkletCode], {\n        type: 'application/javascript',\n      })\n      const workletUrl = URL.createObjectURL(workletBlob)\n\n      // Load the audio worklet module, then release the blob URL.\n      await this.audioContext.audioWorklet.addModule(workletUrl)\n      URL.revokeObjectURL(workletUrl)\n\n      // Create worklet node\n      this.workletNode = new AudioWorkletNode(\n        this.audioContext,\n        'pcm-processor',\n      )\n\n      // Create gain node for volume control\n      this.gainNode = this.audioContext.createGain()\n      this.gainNode.gain.value = this.volume\n\n      // Create analyser for volume detection\n      this.analyser = this.audioContext.createAnalyser()\n      this.analyser.fftSize = 2048 // Larger size for more accurate level detection\n      this.analyser.smoothingTimeConstant = 0.3\n\n      // Connect nodes\n      this.workletNode.connect(this.gainNode)\n      this.gainNode.connect(this.analyser)\n      this.analyser.connect(this.audioContext.destination)\n\n      this.isInitialized = true\n    } catch (error) {\n      // Release the audio context if initialization failed partway through.\n      this.destroy()\n      throw error\n    }\n  }\n\n  async play(pcmData: ArrayBuffer) {\n    if (!this.isInitialized) {\n      await this.init()\n    }\n\n    // Resume audio context if suspended\n    if (this.audioContext?.state === 'suspended') {\n      await this.audioContext.resume()\n    }\n\n    // Convert PCM16 LE to Float32\n    const inputArray = new Int16Array(pcmData)\n    const float32Data = new Float32Array(inputArray.length)\n    for (let i = 0; i < inputArray.length; i++) {\n      float32Data[i] = (inputArray[i] ?? 0) / 32768\n    }\n\n    // Send to worklet for playback\n    this.workletNode?.port.postMessage(float32Data)\n  }\n\n  /* Interrupt playback */\n  interrupt() {\n    if (this.workletNode) {\n      this.workletNode.port.postMessage('interrupt')\n    }\n  }\n\n  setVolume(volume: number) {\n    this.volume = Math.max(0, Math.min(1, volume))\n    if (this.gainNode) {\n      this.gainNode.gain.value = this.volume\n    }\n  }\n\n  destroy() {\n    if (this.audioContext) {\n      void this.audioContext.close()\n      this.audioContext = null\n    }\n    this.isInitialized = false\n  }\n}\n"],"mappings":";;;;AAKA,IAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8C3B,IAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+D5B,SAAS,eAAe,UAAgC;CACtD,MAAM,OAAO,IAAI,WAAW,SAAS,OAAO;CAC5C,SAAS,sBAAsB,IAAI;CAInC,IAAI,eAAe;CACnB,KAAK,MAAM,UAAU,MAAM;EACzB,MAAM,YAAY,KAAK,IAAI,SAAS,GAAG;EACvC,IAAI,YAAY,cACd,eAAe;CAEnB;CAKA,MAAM,aAAa,eAAe;CAClC,OAAO,KAAK,IAAI,GAAG,aAAa,GAAG;AACrC;AAEA,SAAgB,oBAAoB,QAA6B;CAC/D,MAAM,SAAS,KAAK,MAAM;CAE1B,OADc,WAAW,KAAK,SAAS,SAAS,KAAK,WAAW,CAAC,CAC1D,CAAA,CAAM;AACf;AAIA,IAAM,qCAAqB,IAAI,WAAW,IAAI;AAC9C,IAAM,uCAAsB,IAAI,WAAW,IAAI,EAAA,CAAE,KAAK,GAAG;AAEzD,IAAa,gBAAb,MAA2B;CACzB,eAA4C;CAC5C,eAAgD;CAChD,cAA0C;CAC1C,WAAwC;CACxC,cAAsB;CACtB,aAA8B;CAC9B,SAAmD;CAEnD,YAAY,QAA0B;EACpC,KAAK,SAAS;CAChB;CAEA,IAAI,aAAa;EACf,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,OAAO,eAAe,KAAK,QAAQ;CACrC;CAEA,IAAI,qBAAqB;EACvB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;EAC3D,KAAK,SAAS,qBAAqB,IAAI;EACvC,OAAO;CACT;CAEA,IAAI,sBAAsB;EACxB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;EACjD,KAAK,SAAS,sBAAsB,IAAI;EACxC,OAAO;CACT;CAEA,IAAI,kBAAkB;EACpB,OAAO,KAAK;CACd;CAEA,MAAM,QAAQ;EACZ,IAAI;GACF,MAAM,mBAA0C;IAC9C,YAAY,KAAK;IACjB,kBAAkB;IAClB,kBAAkB;IAClB,iBAAiB;GACnB;GAGA,KAAK,cAAc,MAAM,UAAU,aAAa,aAAa,EAC3D,OAAO,iBACT,CAAC;GAMD,KAHc,KAAK,YAAY,eAAe,CAAC,CAAC,EAC/B,EAAO,YAAY,EAAA,EAEtB,iBACZ,QAAQ,KAAK,2BAA2B;GAI1C,KAAK,eAAe,IAAI,aAAa,EACnC,YAAY,KAAK,WACnB,CAAC;GAED,IAAI,KAAK,aAAa,UAAU,aAC9B,MAAM,KAAK,aAAa,OAAO;GAGjC,MAAM,cAAc,IAAI,KAAK,CAAC,kBAAkB,GAAG,EACjD,MAAM,yBACR,CAAC;GACD,MAAM,aAAa,IAAI,gBAAgB,WAAW;GAGlD,MAAM,KAAK,aAAa,aAAa,UAAU,UAAU;GACzD,IAAI,gBAAgB,UAAU;GAG9B,KAAK,eAAe,IAAI,iBACtB,KAAK,cACL,yBACF;GAGA,KAAK,aAAa,KAAK,aAAa,UAAU;IAC5C,IAAI,CAAC,KAAK,aAAa;IAEvB,IAAI,MAAM,KAAK,SAAS,SAAS;KAC/B,MAAM,YAAY,MAAM,KAAK;KAC7B,MAAM,UAAU,KAAK,eAAe,SAAS;KAC7C,MAAM,cAAc,KAAK,oBAAoB,OAAO;KAGpD,IAAI,KAAK,QAAQ,iBACf,KAAK,OAAO,iBAAiB,WAAW;IAE5C;GACF;GAGA,KAAK,WAAW,KAAK,aAAa,eAAe;GACjD,KAAK,SAAS,UAAU;GACxB,KAAK,SAAS,wBAAwB;GAItC,KADoB,aAAa,wBAAwB,KAAK,WAC9D,CAAA,CAAO,QAAQ,KAAK,QAAQ;GAC5B,KAAK,SAAS,QAAQ,KAAK,YAAY;GAGvC,KAAK,cAAc;EACrB,SAAS,OAAO;GAEd,KAAK,KAAK;GACV,MAAM;EACR;CACF;CAEA,OAAO;EACL,KAAK,cAAc;EAEnB,IAAI,KAAK,cAAc;GACrB,KAAK,aAAa,WAAW;GAC7B,KAAK,aAAa,KAAK,MAAM;GAC7B,KAAK,eAAe;EACtB;EAEA,IAAI,KAAK,cAAc;GACrB,KAAU,aAAa,MAAM;GAC7B,KAAK,eAAe;EACtB;EAEA,IAAI,KAAK,aAAa;GACpB,KAAK,YAAY,UAAU,CAAC,CAAC,SAAS,UAAU,MAAM,KAAK,CAAC;GAC5D,KAAK,cAAc;EACrB;CACF;CAEA,oBAAoB;EAClB,IAAI,KAAK,aACP,KAAK,MAAM,SAAS,KAAK,YAAY,eAAe,GAClD,MAAM,UAAU;EAGpB,KAAK,cAAc;CACrB;CAEA,mBAAmB;EACjB,IAAI,KAAK,aAEP,KAAK,MAAM,SAAS,KAAK,YAAY,eAAe,GAClD,MAAM,UAAU;EAGpB,KAAK,cAAc;CACrB;CAEA,eAAuB,cAAyC;EAC9D,MAAM,aAAa,IAAI,WAAW,aAAa,MAAM;EACrD,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,QAAQ,KAEvC,WAAW,KADI,KAAK,IAAI,IAAI,KAAK,IAAI,GAAG,aAAa,MAAM,CAAC,CAC5C,IAAS;EAE3B,OAAO,WAAW;CACpB;CAEA,oBAA4B,QAA6B;EACvD,MAAM,QAAQ,IAAI,WAAW,MAAM;EACnC,MAAM,SAAS,OAAO,aAAa,GAAG,KAAK;EAC3C,OAAO,KAAK,MAAM;CACpB;AACF;AAEA,IAAa,cAAb,MAAyB;CACvB,eAA4C;CAC5C,cAA+C;CAC/C,WAAoC;CACpC,WAAwC;CACxC,gBAAwB;CACxB,SAAiB;CACjB,aAA8B;CAE9B,IAAI,cAAc;EAChB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,OAAO,eAAe,KAAK,QAAQ;CACrC;CAEA,IAAI,sBAAsB;EACxB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,iBAAiB;EAC3D,KAAK,SAAS,qBAAqB,IAAI;EACvC,OAAO;CACT;CAEA,IAAI,uBAAuB;EACzB,IAAI,CAAC,KAAK,UAAU,OAAO;EAC3B,MAAM,OAAO,IAAI,WAAW,KAAK,SAAS,OAAO;EACjD,KAAK,SAAS,sBAAsB,IAAI;EACxC,OAAO;CACT;CAEA,IAAI,mBAAmB;EACrB,OAAO,KAAK;CACd;CAEA,MAAM,OAAO;EACX,IAAI,KAAK,eAAe;EAExB,IAAI;GAEF,KAAK,eAAe,IAAI,aAAa,EACnC,YAAY,KAAK,WACnB,CAAC;GAED,MAAM,cAAc,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAClD,MAAM,yBACR,CAAC;GACD,MAAM,aAAa,IAAI,gBAAgB,WAAW;GAGlD,MAAM,KAAK,aAAa,aAAa,UAAU,UAAU;GACzD,IAAI,gBAAgB,UAAU;GAG9B,KAAK,cAAc,IAAI,iBACrB,KAAK,cACL,eACF;GAGA,KAAK,WAAW,KAAK,aAAa,WAAW;GAC7C,KAAK,SAAS,KAAK,QAAQ,KAAK;GAGhC,KAAK,WAAW,KAAK,aAAa,eAAe;GACjD,KAAK,SAAS,UAAU;GACxB,KAAK,SAAS,wBAAwB;GAGtC,KAAK,YAAY,QAAQ,KAAK,QAAQ;GACtC,KAAK,SAAS,QAAQ,KAAK,QAAQ;GACnC,KAAK,SAAS,QAAQ,KAAK,aAAa,WAAW;GAEnD,KAAK,gBAAgB;EACvB,SAAS,OAAO;GAEd,KAAK,QAAQ;GACb,MAAM;EACR;CACF;CAEA,MAAM,KAAK,SAAsB;EAC/B,IAAI,CAAC,KAAK,eACR,MAAM,KAAK,KAAK;EAIlB,IAAI,KAAK,cAAc,UAAU,aAC/B,MAAM,KAAK,aAAa,OAAO;EAIjC,MAAM,aAAa,IAAI,WAAW,OAAO;EACzC,MAAM,cAAc,IAAI,aAAa,WAAW,MAAM;EACtD,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KACrC,YAAY,MAAM,WAAW,MAAM,KAAK;EAI1C,KAAK,aAAa,KAAK,YAAY,WAAW;CAChD;CAGA,YAAY;EACV,IAAI,KAAK,aACP,KAAK,YAAY,KAAK,YAAY,WAAW;CAEjD;CAEA,UAAU,QAAgB;EACxB,KAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;EAC7C,IAAI,KAAK,UACP,KAAK,SAAS,KAAK,QAAQ,KAAK;CAEpC;CAEA,UAAU;EACR,IAAI,KAAK,cAAc;GACrB,KAAU,aAAa,MAAM;GAC7B,KAAK,eAAe;EACtB;EACA,KAAK,gBAAgB;CACvB;AACF"}

package/dist/esm/tools/code-execution-tool.js

@@ -1,18 +1,16 @@
 import { brandProviderTool } from "@tanstack/ai";
+//#region src/tools/code-execution-tool.ts
 function convertCodeExecutionToolToAdapterFormat(_tool) {
-  return {
-    codeExecution: {}
-  };
+	return { codeExecution: {} };
 }
 function codeExecutionTool() {
-  return brandProviderTool({
-    name: "code_execution",
-    description: "",
-    metadata: {}
-  });
+	return brandProviderTool({
+		name: "code_execution",
+		description: "",
+		metadata: {}
+	});
 }
-export {
-  codeExecutionTool,
-  convertCodeExecutionToolToAdapterFormat
-};
-//# sourceMappingURL=code-execution-tool.js.map
+//#endregion
+export { codeExecutionTool, convertCodeExecutionToolToAdapterFormat };
+
+//# sourceMappingURL=code-execution-tool.js.map

package/dist/esm/tools/code-execution-tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-execution-tool.js","sources":["../../../src/tools/code-execution-tool.ts"],"sourcesContent":["import { brandProviderTool } from '@tanstack/ai'\nimport type { ProviderTool, Tool } from '@tanstack/ai'\n\nexport interface CodeExecutionToolConfig {}\n\n/** @deprecated Renamed to `CodeExecutionToolConfig`. Will be removed in a future release. */\nexport type CodeExecutionTool = CodeExecutionToolConfig\n\nexport type GeminiCodeExecutionTool = ProviderTool<'gemini', 'code_execution'>\n\nexport function convertCodeExecutionToolToAdapterFormat(_tool: Tool) {\n  return {\n    codeExecution: {},\n  }\n}\n\nexport function codeExecutionTool(): GeminiCodeExecutionTool {\n  return brandProviderTool<GeminiCodeExecutionTool>({\n    name: 'code_execution',\n    description: '',\n    metadata: {},\n  })\n}\n"],"names":[],"mappings":";AAUO,SAAS,wCAAwC,OAAa;AACnE,SAAO;AAAA,IACL,eAAe,CAAA;AAAA,EAAC;AAEpB;AAEO,SAAS,oBAA6C;AAC3D,SAAO,kBAA2C;AAAA,IAChD,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU,CAAA;AAAA,EAAC,CACZ;AACH;"}
+{"version":3,"file":"code-execution-tool.js","names":[],"sources":["../../../src/tools/code-execution-tool.ts"],"sourcesContent":["import { brandProviderTool } from '@tanstack/ai'\nimport type { ProviderTool, Tool } from '@tanstack/ai'\n\nexport interface CodeExecutionToolConfig {}\n\n/** @deprecated Renamed to `CodeExecutionToolConfig`. Will be removed in a future release. */\nexport type CodeExecutionTool = CodeExecutionToolConfig\n\nexport type GeminiCodeExecutionTool = ProviderTool<'gemini', 'code_execution'>\n\nexport function convertCodeExecutionToolToAdapterFormat(_tool: Tool) {\n  return {\n    codeExecution: {},\n  }\n}\n\nexport function codeExecutionTool(): GeminiCodeExecutionTool {\n  return brandProviderTool<GeminiCodeExecutionTool>({\n    name: 'code_execution',\n    description: '',\n    metadata: {},\n  })\n}\n"],"mappings":";;AAUA,SAAgB,wCAAwC,OAAa;CACnE,OAAO,EACL,eAAe,CAAC,EAClB;AACF;AAEA,SAAgB,oBAA6C;CAC3D,OAAO,kBAA2C;EAChD,MAAM;EACN,aAAa;EACb,UAAU,CAAC;CACb,CAAC;AACH"}

package/dist/esm/tools/computer-use-tool.js

@@ -1,33 +1,23 @@
 import { brandProviderTool } from "@tanstack/ai";
+//#region src/tools/computer-use-tool.ts
 function convertComputerUseToolToAdapterFormat(tool) {
-  const metadata = tool.metadata;
-  return {
-    computerUse: {
-      ...metadata.environment !== void 0 && {
-        environment: metadata.environment
-      },
-      ...metadata.excludedPredefinedFunctions !== void 0 && {
-        excludedPredefinedFunctions: metadata.excludedPredefinedFunctions
-      }
-    }
-  };
+	const metadata = tool.metadata;
+	return { computerUse: {
+		...metadata.environment !== void 0 && { environment: metadata.environment },
+		...metadata.excludedPredefinedFunctions !== void 0 && { excludedPredefinedFunctions: metadata.excludedPredefinedFunctions }
+	} };
 }
 function computerUseTool(config) {
-  return brandProviderTool({
-    name: "computer_use",
-    description: "",
-    metadata: {
-      ...config.environment !== void 0 && {
-        environment: config.environment
-      },
-      ...config.excludedPredefinedFunctions !== void 0 && {
-        excludedPredefinedFunctions: config.excludedPredefinedFunctions
-      }
-    }
-  });
+	return brandProviderTool({
+		name: "computer_use",
+		description: "",
+		metadata: {
+			...config.environment !== void 0 && { environment: config.environment },
+			...config.excludedPredefinedFunctions !== void 0 && { excludedPredefinedFunctions: config.excludedPredefinedFunctions }
+		}
+	});
 }
-export {
-  computerUseTool,
-  convertComputerUseToolToAdapterFormat
-};
-//# sourceMappingURL=computer-use-tool.js.map
+//#endregion
+export { computerUseTool, convertComputerUseToolToAdapterFormat };
+
+//# sourceMappingURL=computer-use-tool.js.map