@tanstack/ai-gemini 0.19.1 → 0.20.0

package/dist/esm/realtime/client.js

@@ -0,0 +1,389 @@
+import { convertSchemaToJsonSchema } from "@tanstack/ai";
+import { Modality, StartSensitivity, EndSensitivity, ActivityHandling, TurnCoverage } from "@google/genai";
+function clientToolToDeclaration(tool) {
+  return {
+    name: tool.name,
+    description: tool.description,
+    parametersJsonSchema: convertSchemaToJsonSchema(tool.inputSchema),
+    responseJsonSchema: convertSchemaToJsonSchema(tool.outputSchema)
+  };
+}
+function toolConfigToDeclaration(tool) {
+  return {
+    name: tool.name,
+    description: tool.description,
+    parametersJsonSchema: tool.inputSchema,
+    responseJsonSchema: tool.outputSchema
+  };
+}
+function parseResponseMessages(data) {
+  const responses = [];
+  const serverContent = data.serverContent;
+  const parts = serverContent?.modelTurn?.parts;
+  if (data.setupComplete) {
+    responses.push({ type: "setup_complete", data: "", endOfTurn: false });
+    return responses;
+  }
+  if (data.toolCall) {
+    responses.push({
+      type: "tool_call",
+      data: data.toolCall,
+      endOfTurn: false
+    });
+    return responses;
+  }
+  if (data.sessionResumptionUpdate) {
+    responses.push({
+      type: "session_resumption_update",
+      data: data.sessionResumptionUpdate,
+      endOfTurn: false
+    });
+  }
+  if (data.goAway) {
+    responses.push({ type: "go_away", data: data.goAway, endOfTurn: false });
+  }
+  if (data.usageMetadata) {
+    responses.push({
+      type: "usage_metadata",
+      data: data.usageMetadata,
+      endOfTurn: false
+    });
+  }
+  if (parts?.length) {
+    for (const part of parts) {
+      if (part.inlineData?.data) {
+        responses.push({
+          type: "audio",
+          data: {
+            audioData: part.inlineData.data,
+            // The transcription is independent to the model turn, which means it doesn't imply any ordering between transcription and model turn.
+            transcript: ""
+          },
+          endOfTurn: false
+        });
+      } else if (part.text) {
+        responses.push({
+          type: part.thought ? "thought" : "text",
+          data: part.text,
+          endOfTurn: false
+        });
+      }
+    }
+  }
+  if (serverContent?.inputTranscription) {
+    responses.push({
+      type: "input_transcription",
+      data: {
+        text: serverContent.inputTranscription.text || "",
+        finished: serverContent.inputTranscription.finished || false
+      },
+      endOfTurn: false
+    });
+  }
+  if (serverContent?.outputTranscription) {
+    responses.push({
+      type: "output_transcription",
+      data: {
+        text: serverContent.outputTranscription.text || "",
+        finished: serverContent.outputTranscription.finished || false
+      },
+      endOfTurn: false
+    });
+  }
+  if (serverContent?.interrupted) {
+    responses.push({ type: "interrupted", data: "", endOfTurn: false });
+  }
+  if (serverContent?.turnComplete) {
+    responses.push({ type: "turn_complete", data: "", endOfTurn: true });
+  }
+  return responses;
+}
+class GeminiLiveClient {
+  token = null;
+  model = null;
+  responseModalities = [Modality.AUDIO];
+  systemInstructions = "";
+  googleGrounding = false;
+  voiceName = "Puck";
+  temperature = 1;
+  inputAudioTranscription = false;
+  outputAudioTranscription = false;
+  contextWindowCompression;
+  proactiveAudio = false;
+  enableAffectiveDialog = false;
+  thinkingConfig;
+  speechLanguageCode;
+  maxOutputTokens;
+  functionDeclarations = [];
+  automaticActivityDetection = {
+    disabled: false,
+    silence_duration_ms: 2e3,
+    prefix_padding_ms: 500,
+    end_of_speech_sensitivity: EndSensitivity.END_SENSITIVITY_UNSPECIFIED,
+    start_of_speech_sensitivity: StartSensitivity.START_SENSITIVITY_UNSPECIFIED
+  };
+  activityHandling = ActivityHandling.ACTIVITY_HANDLING_UNSPECIFIED;
+  webSocket = null;
+  lastResumptionUpdate = null;
+  setupComplete = false;
+  connected = false;
+  onReceiveResponse = () => {
+  };
+  onOpen = () => {
+  };
+  onClose = () => {
+  };
+  onError = () => {
+  };
+  constructor(token, model, tools) {
+    this.token = token;
+    this.model = model;
+    if (tools) {
+      this.functionDeclarations = tools.map(clientToolToDeclaration);
+    }
+  }
+  get isConnected() {
+    return this.connected;
+  }
+  get isSetupComplete() {
+    return this.setupComplete;
+  }
+  /**
+   * Connection management
+   */
+  connect() {
+    return new Promise((resolve, reject) => {
+      const socket = new WebSocket(
+        `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained?access_token=${this.token}`
+      );
+      this.webSocket = socket;
+      let errored = false;
+      socket.onclose = () => {
+        this.connected = false;
+        this.setupComplete = false;
+        if (!errored) this.onClose();
+        reject(new Error("WebSocket closed before setup completed"));
+      };
+      socket.onerror = () => {
+        errored = true;
+        this.connected = false;
+        this.setupComplete = false;
+        const error = new Error("Gemini realtime WebSocket connection error");
+        this.onError(error);
+        reject(error);
+      };
+      socket.onopen = () => {
+        this.connected = true;
+        this.onOpen();
+        resolve();
+      };
+      socket.onmessage = (event) => {
+        void this.onReceiveMessage(event);
+      };
+    });
+  }
+  disconnect() {
+    if (this.webSocket) {
+      this.webSocket.onclose = null;
+      this.webSocket.onerror = null;
+      this.webSocket.onopen = null;
+      this.webSocket.onmessage = null;
+      this.webSocket.close();
+      this.webSocket = null;
+    }
+    this.connected = false;
+    this.setupComplete = false;
+  }
+  /**
+   * Session management
+   */
+  sendInitialSetupMessage(resume = false) {
+    const tools = this.functionDeclarations;
+    const setup = {
+      model: `models/${this.model}`,
+      generationConfig: {
+        responseModalities: this.responseModalities,
+        temperature: this.temperature,
+        speechConfig: {
+          languageCode: this.speechLanguageCode,
+          voiceConfig: {
+            prebuiltVoiceConfig: {
+              voiceName: this.voiceName
+            }
+          }
+        },
+        enableAffectiveDialog: this.enableAffectiveDialog,
+        maxOutputTokens: this.maxOutputTokens,
+        thinkingConfig: this.thinkingConfig
+      },
+      sessionResumption: {
+        transparent: true,
+        handle: resume ? this.lastResumptionUpdate?.newHandle : void 0
+      },
+      contextWindowCompression: this.contextWindowCompression,
+      proactivity: {
+        proactiveAudio: this.proactiveAudio
+      },
+      systemInstruction: { parts: [{ text: this.systemInstructions }] },
+      tools: [{ functionDeclarations: tools }],
+      realtimeInputConfig: {
+        automaticActivityDetection: {
+          disabled: this.automaticActivityDetection.disabled,
+          silenceDurationMs: this.automaticActivityDetection.silence_duration_ms,
+          prefixPaddingMs: this.automaticActivityDetection.prefix_padding_ms,
+          endOfSpeechSensitivity: this.automaticActivityDetection.end_of_speech_sensitivity,
+          startOfSpeechSensitivity: this.automaticActivityDetection.start_of_speech_sensitivity
+        },
+        activityHandling: this.activityHandling,
+        turnCoverage: TurnCoverage.TURN_INCLUDES_ONLY_ACTIVITY
+      }
+    };
+    if (this.inputAudioTranscription) {
+      setup.inputAudioTranscription = {};
+    }
+    if (this.outputAudioTranscription) {
+      setup.outputAudioTranscription = {};
+    }
+    if (this.googleGrounding) {
+      console.warn(
+        "Google Grounding enabled, removing custom function calls if any."
+      );
+      setup.tools = [{ googleSearch: {} }];
+    }
+    this.sendMessage({ setup });
+  }
+  async restartSession(resume = false) {
+    this.disconnect();
+    await this.connect();
+    this.sendInitialSetupMessage(resume);
+  }
+  updateToken(token) {
+    this.token = token.token;
+    const resume = !(token.config.model && this.model != token.config.model);
+    if (!resume) {
+      this.model = token.config.model;
+    }
+    this.restartSession(resume).catch(
+      (err) => this.onError(err instanceof Error ? err : new Error(String(err)))
+    );
+  }
+  async updateSession(config) {
+    if (config.model && !this.setupComplete) {
+      this.model = config.model;
+    }
+    if (config.instructions) {
+      this.systemInstructions = config.instructions;
+    }
+    if (config.tools) {
+      this.functionDeclarations = config.tools.map(toolConfigToDeclaration);
+    }
+    if (config.maxOutputTokens) {
+      this.maxOutputTokens = typeof config.maxOutputTokens === "number" ? config.maxOutputTokens : void 0;
+    }
+    if (config.temperature) {
+      this.temperature = config.temperature;
+    }
+    if (config.voice) {
+      this.voiceName = config.voice;
+    }
+    const providerOptions = config.providerOptions;
+    if (providerOptions?.googleGrounding) {
+      this.googleGrounding = providerOptions.googleGrounding;
+    }
+    if (providerOptions?.proactiveAudio) {
+      this.proactiveAudio = providerOptions.proactiveAudio;
+    }
+    if (providerOptions?.enableAffectiveDialog) {
+      this.enableAffectiveDialog = providerOptions.enableAffectiveDialog;
+    }
+    if (providerOptions?.contextWindowCompression) {
+      this.contextWindowCompression = providerOptions.contextWindowCompression;
+    }
+    if (providerOptions?.thinkingConfig) {
+      this.thinkingConfig = providerOptions.thinkingConfig;
+    }
+    if (providerOptions?.languageCode) {
+      this.speechLanguageCode = providerOptions.languageCode;
+    }
+    const includeTranscription = config.outputModalities?.includes("text") || false;
+    this.inputAudioTranscription = includeTranscription;
+    this.outputAudioTranscription = includeTranscription;
+    if (!this.setupComplete) {
+      this.sendInitialSetupMessage();
+    } else {
+      return this.restartSession(true);
+    }
+  }
+  /**
+   * Message transmission & receiving
+   */
+  sendMessage(message) {
+    if (this.webSocket?.readyState === WebSocket.OPEN) {
+      this.webSocket.send(JSON.stringify(message));
+    } else {
+      this.onError(
+        new Error("Cannot send message: Gemini realtime socket is not open")
+      );
+    }
+  }
+  async onReceiveMessage(messageEvent) {
+    let jsonData;
+    if (messageEvent.data instanceof Blob) {
+      jsonData = await messageEvent.data.text();
+    } else if (messageEvent.data instanceof ArrayBuffer) {
+      jsonData = new TextDecoder().decode(messageEvent.data);
+    } else {
+      jsonData = messageEvent.data;
+    }
+    try {
+      const messageData = JSON.parse(jsonData);
+      const responses = parseResponseMessages(messageData);
+      for (const response of responses) {
+        if (response.type === "session_resumption_update" && response.data.resumable) {
+          this.lastResumptionUpdate = response.data;
+        }
+        if (response.type === "setup_complete") {
+          this.setupComplete = true;
+        }
+        this.onReceiveResponse(response);
+      }
+    } catch (err) {
+      this.onError(err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+  sendRealtimeInputMessage(data, mimeType) {
+    const blob = { mimeType, data };
+    if (mimeType.startsWith("audio/")) {
+      this.sendMessage({ realtimeInput: { audio: blob } });
+    } else if (mimeType.startsWith("image/") || mimeType.startsWith("video/")) {
+      this.sendMessage({ realtimeInput: { video: blob } });
+    }
+  }
+  sendAudioMessage(base64PCM) {
+    this.sendRealtimeInputMessage(base64PCM, "audio/pcm");
+  }
+  sendImageMessage(base64, mimeType = "image/jpeg") {
+    this.sendRealtimeInputMessage(base64, mimeType);
+  }
+  sendTextMessage(text) {
+    const message = {
+      realtimeInput: {
+        text
+      }
+    };
+    this.sendMessage(message);
+  }
+  sendToolResponse(functionResponses) {
+    const message = {
+      toolResponse: {
+        functionResponses
+      }
+    };
+    this.sendMessage(message);
+  }
+}
+export {
+  GeminiLiveClient,
+  parseResponseMessages
+};
+//# sourceMappingURL=client.js.map

package/dist/esm/realtime/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sources":["../../../src/realtime/client.ts"],"sourcesContent":["import { convertSchemaToJsonSchema } from '@tanstack/ai'\nimport {\n  ActivityHandling,\n  EndSensitivity,\n  Modality,\n  StartSensitivity,\n  TurnCoverage,\n} from '@google/genai'\nimport type {\n  ContextWindowCompressionConfig,\n  FunctionDeclaration,\n  FunctionResponse,\n  LiveClientMessage,\n  LiveServerGoAway,\n  LiveServerMessage,\n  LiveServerSessionResumptionUpdate,\n  LiveServerToolCall,\n  ThinkingConfig,\n  UsageMetadata,\n} from '@google/genai'\nimport type {\n  AnyClientTool,\n  RealtimeSessionConfig,\n  RealtimeToken,\n  RealtimeToolConfig,\n} from '@tanstack/ai'\nimport type {\n  GeminiRealtimeModel,\n  GeminiRealtimeProviderOptions,\n  GeminiRealtimeVoice,\n} from './types'\n\n/** Build a Gemini FunctionDeclaration from an isomorphic client tool (Zod). */\nfunction clientToolToDeclaration(tool: AnyClientTool): FunctionDeclaration {\n  return {\n    name: tool.name,\n    description: tool.description,\n    parametersJsonSchema: convertSchemaToJsonSchema(tool.inputSchema),\n    responseJsonSchema: convertSchemaToJsonSchema(tool.outputSchema),\n  }\n}\n\n/** Build a Gemini FunctionDeclaration from an already-serialized tool config. */\nfunction toolConfigToDeclaration(\n  tool: RealtimeToolConfig,\n): FunctionDeclaration {\n  return {\n    name: tool.name,\n    description: tool.description,\n    parametersJsonSchema: tool.inputSchema,\n    responseJsonSchema: tool.outputSchema,\n  }\n}\n\ninterface LiveResponsePayloads {\n  text: string\n  thought: string\n  audio: { audioData: string; transcript: string }\n  setup_complete: string\n  interrupted: string\n  turn_complete: string\n  tool_call: LiveServerToolCall\n  session_resumption_update: LiveServerSessionResumptionUpdate\n  go_away: LiveServerGoAway\n  usage_metadata: UsageMetadata\n  error: string\n  input_transcription: { text: string; finished: boolean }\n  output_transcription: { text: string; finished: boolean }\n}\n\nexport type MultimodalLiveResponseType = keyof LiveResponsePayloads\n\nexport type LiveResponse = {\n  [K in MultimodalLiveResponseType]: {\n    type: K\n    data: LiveResponsePayloads[K]\n    endOfTurn: boolean\n  }\n}[MultimodalLiveResponseType]\n\n/**\n * Parses response messages from the Gemini Live API\n */\n/**\n * Parses ALL response types from a single server message.\n * The server can now bundle multiple fields (e.g. audio + transcription)\n * in the same message. Returns an array of response objects.\n */\nexport function parseResponseMessages(\n  data: LiveServerMessage,\n): Array<LiveResponse> {\n  const responses: Array<LiveResponse> = []\n  const serverContent = data.serverContent\n  const parts = serverContent?.modelTurn?.parts\n\n  // Setup complete (exclusive — no other fields expected)\n  if (data.setupComplete) {\n    responses.push({ type: 'setup_complete', data: '', endOfTurn: false })\n    return responses\n  }\n\n  // Tool call (exclusive)\n  if (data.toolCall) {\n    responses.push({\n      type: 'tool_call',\n      data: data.toolCall,\n      endOfTurn: false,\n    })\n    return responses\n  }\n\n  if (data.sessionResumptionUpdate) {\n    responses.push({\n      type: 'session_resumption_update',\n      data: data.sessionResumptionUpdate,\n      endOfTurn: false,\n    })\n  }\n\n  if (data.goAway) {\n    responses.push({ type: 'go_away', data: data.goAway, endOfTurn: false })\n  }\n\n  if (data.usageMetadata) {\n    responses.push({\n      type: 'usage_metadata',\n      data: data.usageMetadata,\n      endOfTurn: false,\n    })\n  }\n\n  // Audio data from model turn parts\n  if (parts?.length) {\n    for (const part of parts) {\n      if (part.inlineData?.data) {\n        responses.push({\n          type: 'audio',\n          data: {\n            audioData: part.inlineData.data,\n            // The transcription is independent to the model turn, which means it doesn't imply any ordering between transcription and model turn.\n            transcript: '',\n          },\n          endOfTurn: false,\n        })\n      } else if (part.text) {\n        responses.push({\n          type: part.thought ? 'thought' : 'text',\n          data: part.text,\n          endOfTurn: false,\n        })\n      }\n    }\n  }\n\n  // Transcriptions — checked independently, NOT in else-if with audio\n  if (serverContent?.inputTranscription) {\n    responses.push({\n      type: 'input_transcription',\n      data: {\n        text: serverContent.inputTranscription.text || '',\n        finished: serverContent.inputTranscription.finished || false,\n      },\n      endOfTurn: false,\n    })\n  }\n\n  if (serverContent?.outputTranscription) {\n    responses.push({\n      type: 'output_transcription',\n      data: {\n        text: serverContent.outputTranscription.text || '',\n        finished: serverContent.outputTranscription.finished || false,\n      },\n      endOfTurn: false,\n    })\n  }\n\n  // Interrupted\n  if (serverContent?.interrupted) {\n    responses.push({ type: 'interrupted', data: '', endOfTurn: false })\n  }\n\n  // Turn complete\n  if (serverContent?.turnComplete) {\n    responses.push({ type: 'turn_complete', data: '', endOfTurn: true })\n  }\n\n  return responses\n}\n\nexport class GeminiLiveClient {\n  private token: string | null = null\n  private model: GeminiRealtimeModel | null = null\n\n  private readonly responseModalities: Array<Modality> = [Modality.AUDIO]\n  private systemInstructions = ''\n  private googleGrounding = false\n  private voiceName: GeminiRealtimeVoice = 'Puck'\n  private temperature = 1.0\n  private inputAudioTranscription = false\n  private outputAudioTranscription = false\n  private contextWindowCompression: ContextWindowCompressionConfig | undefined\n  private proactiveAudio = false\n  private enableAffectiveDialog = false\n  private thinkingConfig: ThinkingConfig | undefined\n  private speechLanguageCode: string | undefined\n\n  private maxOutputTokens: number | undefined\n  private functionDeclarations: Array<FunctionDeclaration> = []\n\n  private readonly automaticActivityDetection = {\n    disabled: false,\n    silence_duration_ms: 2000,\n    prefix_padding_ms: 500,\n    end_of_speech_sensitivity: EndSensitivity.END_SENSITIVITY_UNSPECIFIED,\n    start_of_speech_sensitivity: StartSensitivity.START_SENSITIVITY_UNSPECIFIED,\n  }\n\n  private readonly activityHandling =\n    ActivityHandling.ACTIVITY_HANDLING_UNSPECIFIED\n\n  private webSocket: WebSocket | null = null\n  private lastResumptionUpdate: LiveServerSessionResumptionUpdate | null = null\n  private setupComplete = false\n  private connected = false\n\n  public onReceiveResponse: (response: LiveResponse) => void = () => {}\n  public onOpen: () => void = () => {}\n  public onClose: () => void = () => {}\n  public onError: (error: Error) => void = () => {}\n\n  constructor(\n    token: string,\n    model: GeminiRealtimeModel,\n    tools?: ReadonlyArray<AnyClientTool>,\n  ) {\n    this.token = token\n    this.model = model\n\n    if (tools) {\n      this.functionDeclarations = tools.map(clientToolToDeclaration)\n    }\n  }\n\n  get isConnected() {\n    return this.connected\n  }\n\n  get isSetupComplete() {\n    return this.setupComplete\n  }\n\n  /**\n   * Connection management\n   */\n  connect(): Promise<void> {\n    return new Promise((resolve, reject) => {\n      const socket = new WebSocket(\n        `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained?access_token=${this.token}`,\n      )\n      this.webSocket = socket\n\n      // The browser fires `onerror` then `onclose` on an abnormal close; this\n      // flag stops the close handler from overwriting the surfaced error with a\n      // benign \"closed\" signal.\n      let errored = false\n\n      socket.onclose = () => {\n        this.connected = false\n        this.setupComplete = false\n        if (!errored) this.onClose()\n        reject(new Error('WebSocket closed before setup completed'))\n      }\n\n      socket.onerror = () => {\n        errored = true\n        this.connected = false\n        this.setupComplete = false\n        const error = new Error('Gemini realtime WebSocket connection error')\n        this.onError(error)\n        reject(error)\n      }\n\n      socket.onopen = () => {\n        this.connected = true\n        this.onOpen()\n        resolve()\n      }\n\n      socket.onmessage = (event) => {\n        void this.onReceiveMessage(event)\n      }\n    })\n  }\n\n  disconnect() {\n    if (this.webSocket) {\n      // Detach handlers first so a deliberate teardown (e.g. during a\n      // reconnect) doesn't emit a spurious close/error to the client.\n      this.webSocket.onclose = null\n      this.webSocket.onerror = null\n      this.webSocket.onopen = null\n      this.webSocket.onmessage = null\n      this.webSocket.close()\n      this.webSocket = null\n    }\n    this.connected = false\n    this.setupComplete = false\n  }\n\n  /**\n   * Session management\n   */\n  sendInitialSetupMessage(resume = false) {\n    const tools = this.functionDeclarations\n\n    const setup: NonNullable<LiveClientMessage['setup']> = {\n      model: `models/${this.model}`,\n      generationConfig: {\n        responseModalities: this.responseModalities,\n        temperature: this.temperature,\n        speechConfig: {\n          languageCode: this.speechLanguageCode,\n          voiceConfig: {\n            prebuiltVoiceConfig: {\n              voiceName: this.voiceName,\n            },\n          },\n        },\n        enableAffectiveDialog: this.enableAffectiveDialog,\n        maxOutputTokens: this.maxOutputTokens,\n        thinkingConfig: this.thinkingConfig,\n      },\n      sessionResumption: {\n        transparent: true,\n        handle: resume ? this.lastResumptionUpdate?.newHandle : undefined,\n      },\n      contextWindowCompression: this.contextWindowCompression,\n      proactivity: {\n        proactiveAudio: this.proactiveAudio,\n      },\n      systemInstruction: { parts: [{ text: this.systemInstructions }] },\n      tools: [{ functionDeclarations: tools }],\n      realtimeInputConfig: {\n        automaticActivityDetection: {\n          disabled: this.automaticActivityDetection.disabled,\n          silenceDurationMs:\n            this.automaticActivityDetection.silence_duration_ms,\n          prefixPaddingMs: this.automaticActivityDetection.prefix_padding_ms,\n          endOfSpeechSensitivity:\n            this.automaticActivityDetection.end_of_speech_sensitivity,\n          startOfSpeechSensitivity:\n            this.automaticActivityDetection.start_of_speech_sensitivity,\n        },\n        activityHandling: this.activityHandling,\n        turnCoverage: TurnCoverage.TURN_INCLUDES_ONLY_ACTIVITY,\n      },\n    }\n\n    if (this.inputAudioTranscription) {\n      setup.inputAudioTranscription = {}\n    }\n\n    if (this.outputAudioTranscription) {\n      setup.outputAudioTranscription = {}\n    }\n\n    if (this.googleGrounding) {\n      // Currently can't have both Google Search with custom tools.\n      console.warn(\n        'Google Grounding enabled, removing custom function calls if any.',\n      )\n      setup.tools = [{ googleSearch: {} }]\n    }\n\n    this.sendMessage({ setup })\n  }\n\n  async restartSession(resume = false) {\n    this.disconnect()\n    await this.connect()\n    this.sendInitialSetupMessage(resume)\n  }\n\n  updateToken(token: RealtimeToken) {\n    this.token = token.token\n\n    // Restart completely with the new model, or resume the existing session.\n    const resume = !(token.config.model && this.model != token.config.model)\n    if (!resume) {\n      this.model = token.config.model as GeminiRealtimeModel\n    }\n    this.restartSession(resume).catch((err) =>\n      this.onError(err instanceof Error ? err : new Error(String(err))),\n    )\n  }\n\n  async updateSession(config: Partial<RealtimeSessionConfig>) {\n    // model can only be set during initial setup\n    if (config.model && !this.setupComplete) {\n      this.model = config.model as GeminiRealtimeModel\n    }\n\n    if (config.instructions) {\n      this.systemInstructions = config.instructions\n    }\n\n    if (config.tools) {\n      this.functionDeclarations = config.tools.map(toolConfigToDeclaration)\n    }\n\n    if (config.maxOutputTokens) {\n      // Gemini has no \"inf\" sentinel; treat it as \"no explicit limit\".\n      this.maxOutputTokens =\n        typeof config.maxOutputTokens === 'number'\n          ? config.maxOutputTokens\n          : undefined\n    }\n\n    if (config.temperature) {\n      this.temperature = config.temperature\n    }\n\n    if (config.voice) {\n      this.voiceName = config.voice as GeminiRealtimeVoice\n    }\n\n    const providerOptions = config.providerOptions as\n      | GeminiRealtimeProviderOptions\n      | undefined\n\n    if (providerOptions?.googleGrounding) {\n      this.googleGrounding = providerOptions.googleGrounding\n    }\n\n    if (providerOptions?.proactiveAudio) {\n      this.proactiveAudio = providerOptions.proactiveAudio\n    }\n\n    if (providerOptions?.enableAffectiveDialog) {\n      this.enableAffectiveDialog = providerOptions.enableAffectiveDialog\n    }\n\n    if (providerOptions?.contextWindowCompression) {\n      this.contextWindowCompression = providerOptions.contextWindowCompression\n    }\n\n    if (providerOptions?.thinkingConfig) {\n      this.thinkingConfig = providerOptions.thinkingConfig\n    }\n\n    if (providerOptions?.languageCode) {\n      this.speechLanguageCode = providerOptions.languageCode\n    }\n\n    const includeTranscription =\n      config.outputModalities?.includes('text') || false\n    this.inputAudioTranscription = includeTranscription\n    this.outputAudioTranscription = includeTranscription\n\n    if (!this.setupComplete) {\n      this.sendInitialSetupMessage()\n    } else {\n      return this.restartSession(true)\n    }\n  }\n\n  /**\n   * Message transmission & receiving\n   */\n  sendMessage(message: LiveClientMessage) {\n    if (this.webSocket?.readyState === WebSocket.OPEN) {\n      this.webSocket.send(JSON.stringify(message))\n    } else {\n      this.onError(\n        new Error('Cannot send message: Gemini realtime socket is not open'),\n      )\n    }\n  }\n\n  async onReceiveMessage(messageEvent: MessageEvent) {\n    let jsonData\n    if (messageEvent.data instanceof Blob) {\n      jsonData = await messageEvent.data.text()\n    } else if (messageEvent.data instanceof ArrayBuffer) {\n      jsonData = new TextDecoder().decode(messageEvent.data)\n    } else {\n      jsonData = messageEvent.data\n    }\n\n    try {\n      const messageData = JSON.parse(jsonData)\n      // Parse all response types from this message (audio + transcription can coexist)\n      const responses = parseResponseMessages(messageData)\n      for (const response of responses) {\n        if (\n          response.type === 'session_resumption_update' &&\n          response.data.resumable\n        ) {\n          this.lastResumptionUpdate = response.data\n        }\n        if (response.type === 'setup_complete') {\n          this.setupComplete = true\n        }\n        this.onReceiveResponse(response)\n      }\n    } catch (err) {\n      this.onError(err instanceof Error ? err : new Error(String(err)))\n    }\n  }\n\n  sendRealtimeInputMessage(data: string, mimeType: string) {\n    const blob = { mimeType, data }\n\n    if (mimeType.startsWith('audio/')) {\n      this.sendMessage({ realtimeInput: { audio: blob } })\n    } else if (mimeType.startsWith('image/') || mimeType.startsWith('video/')) {\n      this.sendMessage({ realtimeInput: { video: blob } })\n    }\n  }\n\n  sendAudioMessage(base64PCM: string) {\n    this.sendRealtimeInputMessage(base64PCM, 'audio/pcm')\n  }\n\n  sendImageMessage(base64: string, mimeType = 'image/jpeg') {\n    this.sendRealtimeInputMessage(base64, mimeType)\n  }\n\n  sendTextMessage(text: string) {\n    const message: LiveClientMessage = {\n      realtimeInput: {\n        text,\n      },\n    }\n    this.sendMessage(message)\n  }\n\n  sendToolResponse(functionResponses: Array<FunctionResponse>) {\n    const message: LiveClientMessage = {\n      toolResponse: {\n        functionResponses,\n      },\n    }\n    this.sendMessage(message)\n  }\n}\n"],"names":[],"mappings":";;AAiCA,SAAS,wBAAwB,MAA0C;AACzE,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,aAAa,KAAK;AAAA,IAClB,sBAAsB,0BAA0B,KAAK,WAAW;AAAA,IAChE,oBAAoB,0BAA0B,KAAK,YAAY;AAAA,EAAA;AAEnE;AAGA,SAAS,wBACP,MACqB;AACrB,SAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,aAAa,KAAK;AAAA,IAClB,sBAAsB,KAAK;AAAA,IAC3B,oBAAoB,KAAK;AAAA,EAAA;AAE7B;AAoCO,SAAS,sBACd,MACqB;AACrB,QAAM,YAAiC,CAAA;AACvC,QAAM,gBAAgB,KAAK;AAC3B,QAAM,QAAQ,eAAe,WAAW;AAGxC,MAAI,KAAK,eAAe;AACtB,cAAU,KAAK,EAAE,MAAM,kBAAkB,MAAM,IAAI,WAAW,OAAO;AACrE,WAAO;AAAA,EACT;AAGA,MAAI,KAAK,UAAU;AACjB,cAAU,KAAK;AAAA,MACb,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,WAAW;AAAA,IAAA,CACZ;AACD,WAAO;AAAA,EACT;AAEA,MAAI,KAAK,yBAAyB;AAChC,cAAU,KAAK;AAAA,MACb,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,WAAW;AAAA,IAAA,CACZ;AAAA,EACH;AAEA,MAAI,KAAK,QAAQ;AACf,cAAU,KAAK,EAAE,MAAM,WAAW,MAAM,KAAK,QAAQ,WAAW,OAAO;AAAA,EACzE;AAEA,MAAI,KAAK,eAAe;AACtB,cAAU,KAAK;AAAA,MACb,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,WAAW;AAAA,IAAA,CACZ;AAAA,EACH;AAGA,MAAI,OAAO,QAAQ;AACjB,eAAW,QAAQ,OAAO;AACxB,UAAI,KAAK,YAAY,MAAM;AACzB,kBAAU,KAAK;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,YACJ,WAAW,KAAK,WAAW;AAAA;AAAA,YAE3B,YAAY;AAAA,UAAA;AAAA,UAEd,WAAW;AAAA,QAAA,CACZ;AAAA,MACH,WAAW,KAAK,MAAM;AACpB,kBAAU,KAAK;AAAA,UACb,MAAM,KAAK,UAAU,YAAY;AAAA,UACjC,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,QAAA,CACZ;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAGA,MAAI,eAAe,oBAAoB;AACrC,cAAU,KAAK;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,QACJ,MAAM,cAAc,mBAAmB,QAAQ;AAAA,QAC/C,UAAU,cAAc,mBAAmB,YAAY;AAAA,MAAA;AAAA,MAEzD,WAAW;AAAA,IAAA,CACZ;AAAA,EACH;AAEA,MAAI,eAAe,qBAAqB;AACtC,cAAU,KAAK;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,QACJ,MAAM,cAAc,oBAAoB,QAAQ;AAAA,QAChD,UAAU,cAAc,oBAAoB,YAAY;AAAA,MAAA;AAAA,MAE1D,WAAW;AAAA,IAAA,CACZ;AAAA,EACH;AAGA,MAAI,eAAe,aAAa;AAC9B,cAAU,KAAK,EAAE,MAAM,eAAe,MAAM,IAAI,WAAW,OAAO;AAAA,EACpE;AAGA,MAAI,eAAe,cAAc;AAC/B,cAAU,KAAK,EAAE,MAAM,iBAAiB,MAAM,IAAI,WAAW,MAAM;AAAA,EACrE;AAEA,SAAO;AACT;AAEO,MAAM,iBAAiB;AAAA,EACpB,QAAuB;AAAA,EACvB,QAAoC;AAAA,EAE3B,qBAAsC,CAAC,SAAS,KAAK;AAAA,EAC9D,qBAAqB;AAAA,EACrB,kBAAkB;AAAA,EAClB,YAAiC;AAAA,EACjC,cAAc;AAAA,EACd,0BAA0B;AAAA,EAC1B,2BAA2B;AAAA,EAC3B;AAAA,EACA,iBAAiB;AAAA,EACjB,wBAAwB;AAAA,EACxB;AAAA,EACA;AAAA,EAEA;AAAA,EACA,uBAAmD,CAAA;AAAA,EAE1C,6BAA6B;AAAA,IAC5C,UAAU;AAAA,IACV,qBAAqB;AAAA,IACrB,mBAAmB;AAAA,IACnB,2BAA2B,eAAe;AAAA,IAC1C,6BAA6B,iBAAiB;AAAA,EAAA;AAAA,EAG/B,mBACf,iBAAiB;AAAA,EAEX,YAA8B;AAAA,EAC9B,uBAAiE;AAAA,EACjE,gBAAgB;AAAA,EAChB,YAAY;AAAA,EAEb,oBAAsD,MAAM;AAAA,EAAC;AAAA,EAC7D,SAAqB,MAAM;AAAA,EAAC;AAAA,EAC5B,UAAsB,MAAM;AAAA,EAAC;AAAA,EAC7B,UAAkC,MAAM;AAAA,EAAC;AAAA,EAEhD,YACE,OACA,OACA,OACA;AACA,SAAK,QAAQ;AACb,SAAK,QAAQ;AAEb,QAAI,OAAO;AACT,WAAK,uBAAuB,MAAM,IAAI,uBAAuB;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,IAAI,cAAc;AAChB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,UAAyB;AACvB,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,SAAS,IAAI;AAAA,QACjB,iJAAiJ,KAAK,KAAK;AAAA,MAAA;AAE7J,WAAK,YAAY;AAKjB,UAAI,UAAU;AAEd,aAAO,UAAU,MAAM;AACrB,aAAK,YAAY;AACjB,aAAK,gBAAgB;AACrB,YAAI,CAAC,QAAS,MAAK,QAAA;AACnB,eAAO,IAAI,MAAM,yCAAyC,CAAC;AAAA,MAC7D;AAEA,aAAO,UAAU,MAAM;AACrB,kBAAU;AACV,aAAK,YAAY;AACjB,aAAK,gBAAgB;AACrB,cAAM,QAAQ,IAAI,MAAM,4CAA4C;AACpE,aAAK,QAAQ,KAAK;AAClB,eAAO,KAAK;AAAA,MACd;AAEA,aAAO,SAAS,MAAM;AACpB,aAAK,YAAY;AACjB,aAAK,OAAA;AACL,gBAAA;AAAA,MACF;AAEA,aAAO,YAAY,CAAC,UAAU;AAC5B,aAAK,KAAK,iBAAiB,KAAK;AAAA,MAClC;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,aAAa;AACX,QAAI,KAAK,WAAW;AAGlB,WAAK,UAAU,UAAU;AACzB,WAAK,UAAU,UAAU;AACzB,WAAK,UAAU,SAAS;AACxB,WAAK,UAAU,YAAY;AAC3B,WAAK,UAAU,MAAA;AACf,WAAK,YAAY;AAAA,IACnB;AACA,SAAK,YAAY;AACjB,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA;AAAA;AAAA,EAKA,wBAAwB,SAAS,OAAO;AACtC,UAAM,QAAQ,KAAK;AAEnB,UAAM,QAAiD;AAAA,MACrD,OAAO,UAAU,KAAK,KAAK;AAAA,MAC3B,kBAAkB;AAAA,QAChB,oBAAoB,KAAK;AAAA,QACzB,aAAa,KAAK;AAAA,QAClB,cAAc;AAAA,UACZ,cAAc,KAAK;AAAA,UACnB,aAAa;AAAA,YACX,qBAAqB;AAAA,cACnB,WAAW,KAAK;AAAA,YAAA;AAAA,UAClB;AAAA,QACF;AAAA,QAEF,uBAAuB,KAAK;AAAA,QAC5B,iBAAiB,KAAK;AAAA,QACtB,gBAAgB,KAAK;AAAA,MAAA;AAAA,MAEvB,mBAAmB;AAAA,QACjB,aAAa;AAAA,QACb,QAAQ,SAAS,KAAK,sBAAsB,YAAY;AAAA,MAAA;AAAA,MAE1D,0BAA0B,KAAK;AAAA,MAC/B,aAAa;AAAA,QACX,gBAAgB,KAAK;AAAA,MAAA;AAAA,MAEvB,mBAAmB,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,mBAAA,CAAoB,EAAA;AAAA,MAC9D,OAAO,CAAC,EAAE,sBAAsB,OAAO;AAAA,MACvC,qBAAqB;AAAA,QACnB,4BAA4B;AAAA,UAC1B,UAAU,KAAK,2BAA2B;AAAA,UAC1C,mBACE,KAAK,2BAA2B;AAAA,UAClC,iBAAiB,KAAK,2BAA2B;AAAA,UACjD,wBACE,KAAK,2BAA2B;AAAA,UAClC,0BACE,KAAK,2BAA2B;AAAA,QAAA;AAAA,QAEpC,kBAAkB,KAAK;AAAA,QACvB,cAAc,aAAa;AAAA,MAAA;AAAA,IAC7B;AAGF,QAAI,KAAK,yBAAyB;AAChC,YAAM,0BAA0B,CAAA;AAAA,IAClC;AAEA,QAAI,KAAK,0BAA0B;AACjC,YAAM,2BAA2B,CAAA;AAAA,IACnC;AAEA,QAAI,KAAK,iBAAiB;AAExB,cAAQ;AAAA,QACN;AAAA,MAAA;AAEF,YAAM,QAAQ,CAAC,EAAE,cAAc,CAAA,GAAI;AAAA,IACrC;AAEA,SAAK,YAAY,EAAE,OAAO;AAAA,EAC5B;AAAA,EAEA,MAAM,eAAe,SAAS,OAAO;AACnC,SAAK,WAAA;AACL,UAAM,KAAK,QAAA;AACX,SAAK,wBAAwB,MAAM;AAAA,EACrC;AAAA,EAEA,YAAY,OAAsB;AAChC,SAAK,QAAQ,MAAM;AAGnB,UAAM,SAAS,EAAE,MAAM,OAAO,SAAS,KAAK,SAAS,MAAM,OAAO;AAClE,QAAI,CAAC,QAAQ;AACX,WAAK,QAAQ,MAAM,OAAO;AAAA,IAC5B;AACA,SAAK,eAAe,MAAM,EAAE;AAAA,MAAM,CAAC,QACjC,KAAK,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,CAAC;AAAA,IAAA;AAAA,EAEpE;AAAA,EAEA,MAAM,cAAc,QAAwC;AAE1D,QAAI,OAAO,SAAS,CAAC,KAAK,eAAe;AACvC,WAAK,QAAQ,OAAO;AAAA,IACtB;AAEA,QAAI,OAAO,cAAc;AACvB,WAAK,qBAAqB,OAAO;AAAA,IACnC;AAEA,QAAI,OAAO,OAAO;AAChB,WAAK,uBAAuB,OAAO,MAAM,IAAI,uBAAuB;AAAA,IACtE;AAEA,QAAI,OAAO,iBAAiB;AAE1B,WAAK,kBACH,OAAO,OAAO,oBAAoB,WAC9B,OAAO,kBACP;AAAA,IACR;AAEA,QAAI,OAAO,aAAa;AACtB,WAAK,cAAc,OAAO;AAAA,IAC5B;AAEA,QAAI,OAAO,OAAO;AAChB,WAAK,YAAY,OAAO;AAAA,IAC1B;AAEA,UAAM,kBAAkB,OAAO;AAI/B,QAAI,iBAAiB,iBAAiB;AACpC,WAAK,kBAAkB,gBAAgB;AAAA,IACzC;AAEA,QAAI,iBAAiB,gBAAgB;AACnC,WAAK,iBAAiB,gBAAgB;AAAA,IACxC;AAEA,QAAI,iBAAiB,uBAAuB;AAC1C,WAAK,wBAAwB,gBAAgB;AAAA,IAC/C;AAEA,QAAI,iBAAiB,0BAA0B;AAC7C,WAAK,2BAA2B,gBAAgB;AAAA,IAClD;AAEA,QAAI,iBAAiB,gBAAgB;AACnC,WAAK,iBAAiB,gBAAgB;AAAA,IACxC;AAEA,QAAI,iBAAiB,cAAc;AACjC,WAAK,qBAAqB,gBAAgB;AAAA,IAC5C;AAEA,UAAM,uBACJ,OAAO,kBAAkB,SAAS,MAAM,KAAK;AAC/C,SAAK,0BAA0B;AAC/B,SAAK,2BAA2B;AAEhC,QAAI,CAAC,KAAK,eAAe;AACvB,WAAK,wBAAA;AAAA,IACP,OAAO;AACL,aAAO,KAAK,eAAe,IAAI;AAAA,IACjC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,YAAY,SAA4B;AACtC,QAAI,KAAK,WAAW,eAAe,UAAU,MAAM;AACjD,WAAK,UAAU,KAAK,KAAK,UAAU,OAAO,CAAC;AAAA,IAC7C,OAAO;AACL,WAAK;AAAA,QACH,IAAI,MAAM,yDAAyD;AAAA,MAAA;AAAA,IAEvE;AAAA,EACF;AAAA,EAEA,MAAM,iBAAiB,cAA4B;AACjD,QAAI;AACJ,QAAI,aAAa,gBAAgB,MAAM;AACrC,iBAAW,MAAM,aAAa,KAAK,KAAA;AAAA,IACrC,WAAW,aAAa,gBAAgB,aAAa;AACnD,iBAAW,IAAI,YAAA,EAAc,OAAO,aAAa,IAAI;AAAA,IACvD,OAAO;AACL,iBAAW,aAAa;AAAA,IAC1B;AAEA,QAAI;AACF,YAAM,cAAc,KAAK,MAAM,QAAQ;AAEvC,YAAM,YAAY,sBAAsB,WAAW;AACnD,iBAAW,YAAY,WAAW;AAChC,YACE,SAAS,SAAS,+BAClB,SAAS,KAAK,WACd;AACA,eAAK,uBAAuB,SAAS;AAAA,QACvC;AACA,YAAI,SAAS,SAAS,kBAAkB;AACtC,eAAK,gBAAgB;AAAA,QACvB;AACA,aAAK,kBAAkB,QAAQ;AAAA,MACjC;AAAA,IACF,SAAS,KAAK;AACZ,WAAK,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,CAAC;AAAA,IAClE;AAAA,EACF;AAAA,EAEA,yBAAyB,MAAc,UAAkB;AACvD,UAAM,OAAO,EAAE,UAAU,KAAA;AAEzB,QAAI,SAAS,WAAW,QAAQ,GAAG;AACjC,WAAK,YAAY,EAAE,eAAe,EAAE,OAAO,KAAA,GAAQ;AAAA,IACrD,WAAW,SAAS,WAAW,QAAQ,KAAK,SAAS,WAAW,QAAQ,GAAG;AACzE,WAAK,YAAY,EAAE,eAAe,EAAE,OAAO,KAAA,GAAQ;AAAA,IACrD;AAAA,EACF;AAAA,EAEA,iBAAiB,WAAmB;AAClC,SAAK,yBAAyB,WAAW,WAAW;AAAA,EACtD;AAAA,EAEA,iBAAiB,QAAgB,WAAW,cAAc;AACxD,SAAK,yBAAyB,QAAQ,QAAQ;AAAA,EAChD;AAAA,EAEA,gBAAgB,MAAc;AAC5B,UAAM,UAA6B;AAAA,MACjC,eAAe;AAAA,QACb;AAAA,MAAA;AAAA,IACF;AAEF,SAAK,YAAY,OAAO;AAAA,EAC1B;AAAA,EAEA,iBAAiB,mBAA4C;AAC3D,UAAM,UAA6B;AAAA,MACjC,cAAc;AAAA,QACZ;AAAA,MAAA;AAAA,IACF;AAEF,SAAK,YAAY,OAAO;AAAA,EAC1B;AACF;"}

package/dist/esm/realtime/index.d.ts

@@ -0,0 +1,3 @@
+export { geminiRealtimeToken } from './token.js';
+export { geminiRealtime } from './adapter.js';
+export type { GeminiRealtimeModel, GeminiRealtimeVoice, GeminiRealtimeTokenOptions, GeminiRealtimeOptions, GeminiRealtimeProviderOptions, } from './types.js';

package/dist/esm/realtime/token.d.ts

@@ -0,0 +1,26 @@
+import { RealtimeTokenAdapter } from '@tanstack/ai';
+import { GeminiRealtimeTokenOptions } from './types.js';
+/**
+ * Creates a Google Gemini realtime token adapter.
+ *
+ * This adapter generates ephemeral tokens for client-side WebSocket connections.
+ *
+ * @param options - Configuration options for the realtime session
+ * @returns A RealtimeTokenAdapter for use with realtimeToken()
+ *
+ * @example
+ * ```typescript
+ * import { realtimeToken } from '@tanstack/ai'
+ * import { geminiRealtimeToken } from '@tanstack/ai-gemini'
+ *
+ * const token = await realtimeToken({
+ *   adapter: geminiRealtimeToken({
+ *     // Optional: constraint model config by token
+ *     liveConnectConstraints: {
+ *       model: 'gemini-3.1-flash-live-preview',
+ *     },
+ *   }),
+ * })
+ * ```
+ */
+export declare function geminiRealtimeToken(options?: GeminiRealtimeTokenOptions): RealtimeTokenAdapter;

package/dist/esm/realtime/token.js

@@ -0,0 +1,39 @@
+import { GoogleGenAI } from "@google/genai";
+import { getGeminiApiKeyFromEnv } from "../utils/client.js";
+function geminiRealtimeToken(options = {}) {
+  const apiKey = getGeminiApiKeyFromEnv();
+  const client = new GoogleGenAI({
+    apiKey
+  });
+  return {
+    provider: "gemini",
+    async generateToken() {
+      const expireTime = options.expiresAt ?? Date.now() + 30 * 60 * 1e3;
+      const token = await client.authTokens.create({
+        config: {
+          uses: options.uses ?? 1,
+          expireTime: new Date(expireTime).toISOString(),
+          liveConnectConstraints: options.liveConnectConstraints,
+          httpOptions: {
+            apiVersion: "v1alpha"
+          }
+        }
+      });
+      if (!token.name) {
+        throw new Error("Gemini realtime token creation failed");
+      }
+      return {
+        provider: "gemini",
+        token: token.name,
+        expiresAt: expireTime,
+        config: {
+          model: options.liveConnectConstraints?.model
+        }
+      };
+    }
+  };
+}
+export {
+  geminiRealtimeToken
+};
+//# sourceMappingURL=token.js.map

package/dist/esm/realtime/token.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token.js","sources":["../../../src/realtime/token.ts"],"sourcesContent":["import { GoogleGenAI } from '@google/genai'\nimport { getGeminiApiKeyFromEnv } from '../utils'\nimport type { RealtimeToken, RealtimeTokenAdapter } from '@tanstack/ai'\nimport type { GeminiRealtimeTokenOptions } from './types'\n\n/**\n * Creates a Google Gemini realtime token adapter.\n *\n * This adapter generates ephemeral tokens for client-side WebSocket connections.\n *\n * @param options - Configuration options for the realtime session\n * @returns A RealtimeTokenAdapter for use with realtimeToken()\n *\n * @example\n * ```typescript\n * import { realtimeToken } from '@tanstack/ai'\n * import { geminiRealtimeToken } from '@tanstack/ai-gemini'\n *\n * const token = await realtimeToken({\n *   adapter: geminiRealtimeToken({\n *     // Optional: constraint model config by token\n *     liveConnectConstraints: {\n *       model: 'gemini-3.1-flash-live-preview',\n *     },\n *   }),\n * })\n * ```\n */\nexport function geminiRealtimeToken(\n  options: GeminiRealtimeTokenOptions = {},\n): RealtimeTokenAdapter {\n  const apiKey = getGeminiApiKeyFromEnv()\n\n  const client = new GoogleGenAI({\n    apiKey,\n  })\n\n  return {\n    provider: 'gemini',\n    async generateToken(): Promise<RealtimeToken> {\n      // Computed per call so a reused adapter doesn't mint tokens with a\n      // fixed (and eventually past) expiry. Defaults to 30 minutes.\n      const expireTime = options.expiresAt ?? Date.now() + 30 * 60 * 1000\n\n      const token = await client.authTokens.create({\n        config: {\n          uses: options.uses ?? 1,\n          expireTime: new Date(expireTime).toISOString(),\n          liveConnectConstraints: options.liveConnectConstraints,\n          httpOptions: {\n            apiVersion: 'v1alpha',\n          },\n        },\n      })\n\n      if (!token.name) {\n        throw new Error('Gemini realtime token creation failed')\n      }\n\n      return {\n        provider: 'gemini',\n        token: token.name,\n        expiresAt: expireTime,\n        config: {\n          model: options.liveConnectConstraints?.model,\n        },\n      }\n    },\n  }\n}\n"],"names":[],"mappings":";;AA4BO,SAAS,oBACd,UAAsC,IAChB;AACtB,QAAM,SAAS,uBAAA;AAEf,QAAM,SAAS,IAAI,YAAY;AAAA,IAC7B;AAAA,EAAA,CACD;AAED,SAAO;AAAA,IACL,UAAU;AAAA,IACV,MAAM,gBAAwC;AAG5C,YAAM,aAAa,QAAQ,aAAa,KAAK,QAAQ,KAAK,KAAK;AAE/D,YAAM,QAAQ,MAAM,OAAO,WAAW,OAAO;AAAA,QAC3C,QAAQ;AAAA,UACN,MAAM,QAAQ,QAAQ;AAAA,UACtB,YAAY,IAAI,KAAK,UAAU,EAAE,YAAA;AAAA,UACjC,wBAAwB,QAAQ;AAAA,UAChC,aAAa;AAAA,YACX,YAAY;AAAA,UAAA;AAAA,QACd;AAAA,MACF,CACD;AAED,UAAI,CAAC,MAAM,MAAM;AACf,cAAM,IAAI,MAAM,uCAAuC;AAAA,MACzD;AAEA,aAAO;AAAA,QACL,UAAU;AAAA,QACV,OAAO,MAAM;AAAA,QACb,WAAW;AAAA,QACX,QAAQ;AAAA,UACN,OAAO,QAAQ,wBAAwB;AAAA,QAAA;AAAA,MACzC;AAAA,IAEJ;AAAA,EAAA;AAEJ;"}

package/dist/esm/realtime/types.d.ts

@@ -0,0 +1,51 @@
+import { ContextWindowCompressionConfig, LiveConnectConstraints, ThinkingConfig } from '@google/genai';
+/**
+ * Gemini realtime voice options
+ */
+export type GeminiRealtimeVoice = 'Achernar' | 'Achird' | 'Algenib' | 'Algieba' | 'Alnilam' | 'Aoede' | 'Autonoe' | 'Callirrhoe' | 'Charon' | 'Despina' | 'Enceladus' | 'Erinome' | 'Fenrir' | 'Gacrux' | 'Iapetus' | 'Kore' | 'Laomedeia' | 'Leda' | 'Orus' | 'Pulcherrima' | 'Puck' | 'Rasalgethi' | 'Sadachbia' | 'Sadaltager' | 'Schedar' | 'Sulafat' | 'Umbriel' | 'Vindemiatrix' | 'Zephyr' | 'Zubenelgenubi';
+/**
+ * Gemini realtime model options
+ */
+export type GeminiRealtimeModel = 'gemini-3.1-flash-live-preview';
+/**
+ * Options for the Gemini realtime client adapter
+ */
+export interface GeminiRealtimeOptions {
+    /** Connection mode (default: 'websocket' in browser) */
+    connectionMode?: 'websocket';
+    model?: GeminiRealtimeModel;
+}
+export interface StrictLiveConnectionConstraints extends Omit<LiveConnectConstraints, 'model'> {
+    model?: GeminiRealtimeModel;
+}
+/**
+ * Options for the Gemini realtime token adapter
+ */
+export interface GeminiRealtimeTokenOptions {
+    expiresAt?: number;
+    uses?: number;
+    /**
+     * Config for LiveConnectConstraints for Auth Token creation.
+     *
+     * NOTE: Adding liveConnectConstraints will cause the API to ignore any config passed later to WebSocket.
+     */
+    liveConnectConstraints?: StrictLiveConnectionConstraints;
+}
+/**
+ * Gemini-specific realtime options, passed through `providerOptions` on a
+ * realtime session config.
+ */
+export interface GeminiRealtimeProviderOptions {
+    /** Enable Google Search grounding (mutually exclusive with custom tools). */
+    googleGrounding?: boolean;
+    /** Enable proactive audio so the model may choose when to respond. */
+    proactiveAudio?: boolean;
+    /** Enable affective (emotion-aware) dialog. */
+    enableAffectiveDialog?: boolean;
+    /** Context window compression configuration. */
+    contextWindowCompression?: ContextWindowCompressionConfig;
+    /** Thinking configuration. */
+    thinkingConfig?: ThinkingConfig;
+    /** BCP-47 language code for speech output. */
+    languageCode?: string;
+}

package/dist/esm/realtime/utils.d.ts

@@ -0,0 +1,40 @@
+import { GeminiLiveClient } from './client.js';
+export declare function base64ToArrayBuffer(base64: string): ArrayBuffer;
+export declare class AudioStreamer {
+    private audioContext;
+    private audioWorklet;
+    private mediaStream;
+    private analyser;
+    private isStreaming;
+    private readonly sampleRate;
+    private readonly client;
+    constructor(client: GeminiLiveClient);
+    get inputLevel(): number;
+    get inputFrequencyData(): Uint8Array<ArrayBuffer>;
+    get inputTimeDomainData(): Uint8Array<ArrayBuffer>;
+    get inputSampleRate(): number;
+    start(): Promise<void>;
+    stop(): void;
+    startAudioCapture(): void;
+    stopAudioCapture(): void;
+    private convertToPCM16;
+    private arrayBufferToBase64;
+}
+export declare class AudioPlayer {
+    private audioContext;
+    private workletNode;
+    private gainNode;
+    private analyser;
+    private isInitialized;
+    private volume;
+    private readonly sampleRate;
+    get outputLevel(): number;
+    get outputFrequencyData(): Uint8Array<ArrayBuffer>;
+    get outputTimeDomainData(): Uint8Array<ArrayBuffer>;
+    get outputSampleRate(): number;
+    init(): Promise<void>;
+    play(pcmData: ArrayBuffer): Promise<void>;
+    interrupt(): void;
+    setVolume(volume: number): void;
+    destroy(): void;
+}