@livekit/agents-plugin-google 1.0.51 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -162,7 +162,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
162
162
  hasReceivedAudioInput = false;
163
163
  pendingInterruptText = false;
164
164
  earlyCompletionPending = false;
165
- toolCallPending = false;
165
+ pendingToolCallIds = /* @__PURE__ */ new Set();
166
166
  generationPendingTurnComplete;
167
167
  #client;
168
168
  #task;
@@ -208,7 +208,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
208
208
  }
209
209
  this.earlyCompletionPending = false;
210
210
  this.pendingInterruptText = false;
211
- this.toolCallPending = false;
211
+ this.pendingToolCallIds.clear();
212
212
  if (this.generationPendingTurnComplete) {
213
213
  this.markCurrentGenerationDone(false, this.generationPendingTurnComplete);
214
214
  this.generationPendingTurnComplete = void 0;
@@ -337,7 +337,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
337
337
  return ((_b = (_a = this.options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) ?? false;
338
338
  }
339
339
  pushAudio(frame) {
340
- if (this.toolCallPending) return;
340
+ if (this.pendingToolCallIds.size > 0) return;
341
341
  this.hasReceivedAudioInput = true;
342
342
  for (const f of this.resampleAudio(frame)) {
343
343
  for (const nf of this.bstream.write(f.data.buffer)) {
@@ -412,7 +412,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
412
412
  if (!this.manualActivityDetection) {
413
413
  return;
414
414
  }
415
- if (this.toolCallPending) return;
415
+ if (this.pendingToolCallIds.size > 0) return;
416
416
  if (!this.inUserActivity) {
417
417
  this.inUserActivity = true;
418
418
  this.sendClientEvent({
@@ -605,13 +605,15 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
605
605
  functionResponses
606
606
  });
607
607
  } finally {
608
- this.toolCallPending = false;
608
+ for (const fr of functionResponses) {
609
+ if (fr == null ? void 0 : fr.id) this.pendingToolCallIds.delete(fr.id);
610
+ }
609
611
  }
610
612
  }
611
613
  break;
612
614
  case "realtime_input":
613
615
  const { mediaChunks, activityStart, activityEnd, text } = msg.value;
614
- if (this.toolCallPending) break;
616
+ if (this.pendingToolCallIds.size > 0) break;
615
617
  if (mediaChunks) {
616
618
  for (const mediaChunk of mediaChunks) {
617
619
  await session.sendRealtimeInput({ media: mediaChunk });
@@ -646,7 +648,6 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
646
648
  }
647
649
  async onReceiveMessage(session, response) {
648
650
  var _a, _b, _c, _d, _e;
649
- if (response.toolCall) this.toolCallPending = true;
650
651
  const hasAudioData = (_c = (_b = (_a = response.serverContent) == null ? void 0 : _a.modelTurn) == null ? void 0 : _b.parts) == null ? void 0 : _c.some(
651
652
  (part) => {
652
653
  var _a2;
@@ -1011,7 +1012,6 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
1011
1012
  return;
1012
1013
  }
1013
1014
  const gen = this.currentGeneration;
1014
- this.toolCallPending = true;
1015
1015
  if (gen.functionChannel.closed) {
1016
1016
  this.#logger.warn("received tool call but functionChannel is already closed.");
1017
1017
  return;
@@ -1021,15 +1021,16 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
1021
1021
  this.#logger.warn("received function call without name, skipping");
1022
1022
  continue;
1023
1023
  }
1024
+ const callId = fc.id || (0, import_agents.shortuuid)("fnc-call-");
1025
+ this.pendingToolCallIds.add(callId);
1024
1026
  gen.functionChannel.write(
1025
1027
  import_agents.llm.FunctionCall.create({
1026
- callId: fc.id || (0, import_agents.shortuuid)("fnc-call-"),
1028
+ callId,
1027
1029
  name: fc.name,
1028
1030
  args: fc.args ? JSON.stringify(fc.args) : ""
1029
1031
  })
1030
1032
  );
1031
1033
  }
1032
- gen.functionChannel.close();
1033
1034
  }
1034
1035
  handleToolCallCancellation(cancellation) {
1035
1036
  this.#logger.warn(
@@ -1038,6 +1039,9 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
1038
1039
  },
1039
1040
  "server cancelled tool calls"
1040
1041
  );
1042
+ for (const id of cancellation.ids || []) {
1043
+ this.pendingToolCallIds.delete(id);
1044
+ }
1041
1045
  }
1042
1046
  handleUsageMetadata(usage) {
1043
1047
  if (!this.currentGeneration) {
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n normalizeLanguage,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\nconst LK_GOOGLE_DEBUG = Number(process.env.LK_GOOGLE_DEBUG ?? 0);\n\n// WebSocket close codes (RFC 6455)\nconst WS_CLOSE_NORMAL = 1000;\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual<T>(a: Set<T>, b: Set<T>): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel<llm.MessageGeneration>;\n functionChannel: stream.StreamChannel<llm.FunctionCall>;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel<string>;\n audioChannel: stream.StreamChannel<AudioFrame>;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n get model(): string {\n return this._options.model;\n }\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * Thinking configuration for native audio models.\n * If not set, the model's default thinking behavior is used.\n * Use `\\{ thinkingBudget: 0 \\}` to disable thinking.\n * Use `\\{ thinkingBudget: -1 \\}` for automatic/dynamic thinking.\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n manualFunctionCalls: false,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai\n ? 'gemini-live-2.5-flash-native-audio'\n : 'gemini-2.5-flash-native-audio-preview-12-2025';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language ? normalizeLanguage(options.language) : undefined,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise<void> {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue<api_proto.ClientEvents>();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future<llm.GenerationCreatedEvent> } = {};\n private pendingGenerationFut?: Future<llm.GenerationCreatedEvent>;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n private pendingInterruptText = false;\n private earlyCompletionPending = false;\n private toolCallPending = false;\n private generationPendingTurnComplete?: ResponseGeneration;\n\n #client: GoogleGenAI;\n #task: Promise<void>;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise<void> {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n this.earlyCompletionPending = false;\n this.pendingInterruptText = false;\n\n this.toolCallPending = false;\n if (this.generationPendingTurnComplete) {\n this.markCurrentGenerationDone(false, this.generationPendingTurnComplete);\n this.generationPendingTurnComplete = undefined;\n }\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise<void> {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise<void> {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n const shouldSendRealtimeText = this.pendingInterruptText;\n\n if (shouldSendRealtimeText) {\n for (const turn of turns as types.Content[]) {\n if (turn.role !== 'user') continue;\n // Realtime text drives live activity/interrupts\n // { type: content: turnComplete: true } alone does not reliably preempt a streaming response in Gemini Live.\n const text = (turn.parts || [])\n .map((part) => (part as { text?: string }).text)\n .filter((value): value is string => !!value)\n .join('');\n if (text) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: { text },\n });\n this.pendingInterruptText = false;\n }\n }\n }\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise<void> {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n if (this.toolCallPending) return;\n\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent> {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future<llm.GenerationCreatedEvent>();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (this.toolCallPending) return;\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n private generationHasOutput(gen: ResponseGeneration): boolean {\n return Boolean(gen.outputText) || gen._firstTokenTimestamp !== undefined;\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug('interrupt skipped (activityHandling = NO_INTERRUPTION)');\n }\n return;\n }\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.pendingInterruptText = true;\n if (this.generationHasOutput(this.currentGeneration)) {\n this.earlyCompletionPending = true;\n this.markCurrentGenerationDone();\n }\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise<void> {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise<void> {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n // onerror is called for network-level errors (connection refused, DNS failure, TLS errors).\n // Application-level errors (e.g., invalid model name) come through onclose with error codes.\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n // Surface WebSocket close errors to the user instead of silently swallowing them\n if (event.code !== WS_CLOSE_NORMAL) {\n // Note: WebSocket close reasons are limited to 123 bytes by RFC 6455,\n // so Google's error messages may be truncated at the protocol level\n const isTruncated = event.reason && event.reason.length >= 120;\n const truncationNote = isTruncated\n ? ' (message may be truncated - check model name and API permissions)'\n : '';\n const errorMsg = event.reason || `WebSocket closed with code ${event.code}`;\n this.#logger.error(`Gemini Live session error: ${errorMsg}${truncationNote}`);\n\n this.emitError(\n new APIStatusError({\n message: `${errorMsg}${truncationNote}`,\n options: {\n statusCode: event.code,\n retryable: false,\n body: event.reason\n ? { reason: event.reason, code: event.code, truncated: isTruncated }\n : null,\n },\n }),\n false,\n );\n } else {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n }\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise<void> {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n }\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n }\n try {\n await session.sendToolResponse({\n functionResponses,\n });\n } finally {\n this.toolCallPending = false;\n }\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd, text } = msg.value;\n if (this.toolCallPending) break;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (text) {\n await session.sendRealtimeInput({ text });\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise<void> {\n if (response.toolCall) this.toolCallPending = true;\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n } else if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n const shouldStartNewGeneration =\n !this.currentGeneration || this.currentGeneration._done || !!this.pendingGenerationFut;\n if (shouldStartNewGeneration) {\n if (response.serverContent?.interrupted) {\n // Two cases when an interrupted event is sent without an active generation:\n // 1) generation done but playout not finished (turnComplete -> interrupted)\n // 2) generation not started (interrupted -> turnComplete)\n if (!this.pendingGenerationFut) {\n this.handleInputSpeechStarted();\n }\n\n response.serverContent = {\n ...response.serverContent,\n interrupted: undefined,\n };\n\n const sc = response.serverContent;\n const hasServerContent =\n !!sc?.modelTurn ||\n sc?.outputTranscription != null ||\n sc?.inputTranscription != null ||\n sc?.generationComplete != null ||\n sc?.turnComplete != null;\n if (!hasServerContent) {\n response.serverContent = undefined;\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug('ignoring empty server content');\n }\n }\n }\n\n // start new generation for serverContent or for standalone toolCalls\n if (this.isNewGeneration(response)) {\n this.startNewGeneration();\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`new generation started: ${this.currentGeneration?.responseId}`);\n }\n }\n }\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record<string, unknown> {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record<string, unknown> {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(\n keepFunctionChannelOpen: boolean = false,\n gen?: ResponseGeneration,\n ): void {\n const target = gen ?? this.currentGeneration;\n if (!target || target._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const targetGen = target;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (targetGen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: targetGen.inputId,\n transcript: targetGen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: targetGen.inputTranscription,\n id: targetGen.inputId,\n });\n }\n\n if (targetGen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: targetGen.outputText,\n id: targetGen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n targetGen.textChannel.write('');\n }\n\n targetGen.textChannel.close();\n targetGen.audioChannel.close();\n if (!keepFunctionChannelOpen) {\n targetGen.functionChannel.close();\n }\n targetGen.messageChannel.close();\n targetGen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n thinkingConfig: opts.thinkingConfig,\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n const previousGen = this.currentGeneration;\n const previousHadOpenFunctionChannel = previousGen && !previousGen.functionChannel.closed;\n\n // close functionChannel of previous generation if still open (no toolCall arrived)\n if (previousGen && previousHadOpenFunctionChannel) {\n previousGen.functionChannel.close();\n }\n\n if (previousGen && !previousGen._done) {\n if (previousHadOpenFunctionChannel) {\n this.generationPendingTurnComplete = previousGen;\n } else {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel<llm.MessageGeneration>(),\n functionChannel: stream.createStreamChannel<llm.FunctionCall>(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel<string>(),\n audioChannel: stream.createStreamChannel<AudioFrame>(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n responseId,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n const discardOutput = this.earlyCompletionPending;\n\n if (serverContent.modelTurn && !discardOutput) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n // bypass reasoning/thought output\n if (part.thought) {\n continue;\n }\n\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (\n !discardOutput &&\n serverContent.outputTranscription &&\n serverContent.outputTranscription.text\n ) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted && !this.pendingGenerationFut) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete && !this.earlyCompletionPending) {\n if (this.generationPendingTurnComplete) {\n this.markCurrentGenerationDone(false, this.generationPendingTurnComplete);\n this.generationPendingTurnComplete = undefined;\n } else {\n this.markCurrentGenerationDone();\n }\n }\n\n // Assume Gemini emits turnComplete/generationComplete before any new generation content.\n // We keep discarding until that signal to avoid old stream spillover after interrupts.\n if (\n this.earlyCompletionPending &&\n (serverContent.turnComplete || serverContent.generationComplete)\n ) {\n this.earlyCompletionPending = false;\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n this.toolCallPending = true;\n\n if (gen.functionChannel.closed) {\n this.#logger.warn('received tool call but functionChannel is already closed.');\n return;\n }\n\n for (const fc of toolCall.functionCalls || []) {\n if (!fc.name) {\n this.#logger.warn('received function call without name, skipping');\n continue;\n }\n gen.functionChannel.write(\n llm.FunctionCall.create({\n callId: fc.id || shortuuid('fnc-call-'),\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n }),\n );\n }\n\n gen.functionChannel.close();\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() {}\n\n async clearAudio() {}\n\n private *resampleAudio(frame: AudioFrame): Generator<AudioFrame> {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n\n private isNewGeneration(response: types.LiveServerMessage) {\n if (this.earlyCompletionPending) {\n return false;\n }\n if (response.toolCall) {\n return true;\n }\n\n const serverContent = response.serverContent;\n return (\n !!serverContent &&\n (serverContent.modelTurn ||\n serverContent.outputTranscription != null ||\n serverContent.inputTranscription != null ||\n serverContent.generationComplete != null ||\n serverContent.turnComplete != null)\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAgBO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAE9B,MAAM,kBAAkB,OAAO,QAAQ,IAAI,mBAAmB,CAAC;AAG/D,MAAM,kBAAkB;AAIjB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,YACE,UAgJI,CAAC,GACL;AAvSJ;AAwSI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,MAC7D,qBAAqB;AAAA,IACvB,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WACjB,uCACA;AAEJ,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ,eAAW,iCAAkB,QAAQ,QAAQ,IAAI;AAAA,MACnE,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EACxB,uBAAuB;AAAA,EACvB,yBAAyB;AAAA,EACzB,kBAAkB;AAAA,EAClB;AAAA,EAER;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AACA,SAAK,yBAAyB;AAC9B,SAAK,uBAAuB;AAE5B,SAAK,kBAAkB;AACvB,QAAI,KAAK,+BAA+B;AACtC,WAAK,0BAA0B,OAAO,KAAK,6BAA6B;AACxE,WAAK,gCAAgC;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,oBAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,cAAM,yBAAyB,KAAK;AAEpC,YAAI,wBAAwB;AAC1B,qBAAW,QAAQ,OAA0B;AAC3C,gBAAI,KAAK,SAAS,OAAQ;AAG1B,kBAAM,QAAQ,KAAK,SAAS,CAAC,GAC1B,IAAI,CAAC,SAAU,KAA2B,IAAI,EAC9C,OAAO,CAAC,UAA2B,CAAC,CAAC,KAAK,EAC1C,KAAK,EAAE;AACV,gBAAI,MAAM;AACR,mBAAK,gBAAgB;AAAA,gBACnB,MAAM;AAAA,gBACN,OAAO,EAAE,KAAK;AAAA,cAChB,CAAC;AACD,mBAAK,uBAAuB;AAAA,YAC9B;AAAA,UACF;AAAA,QACF;AAEA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzoBzC;AA0oBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AACjC,QAAI,KAAK,gBAAiB;AAG1B,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,KAAK,gBAAiB;AAE1B,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,oBAAoB,KAAkC;AAC5D,WAAO,QAAQ,IAAI,UAAU,KAAK,IAAI,yBAAyB;AAAA,EACjE;AAAA,EAEA,MAAM,YAAY;AA5vBpB;AA8vBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F,UAAI,iBAAiB;AACnB,aAAK,QAAQ,MAAM,wDAAwD;AAAA,MAC7E;AACA;AAAA,IACF;AACA,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,uBAAuB;AAC5B,UAAI,KAAK,oBAAoB,KAAK,iBAAiB,GAAG;AACpD,aAAK,yBAAyB;AAC9B,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA;AAAA;AAAA,YAGA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAE9B,kBAAI,MAAM,SAAS,iBAAiB;AAGlC,sBAAM,cAAc,MAAM,UAAU,MAAM,OAAO,UAAU;AAC3D,sBAAM,iBAAiB,cACnB,uEACA;AACJ,sBAAM,WAAW,MAAM,UAAU,8BAA8B,MAAM,IAAI;AACzE,qBAAK,QAAQ,MAAM,8BAA8B,QAAQ,GAAG,cAAc,EAAE;AAE5E,qBAAK;AAAA,kBACH,IAAI,6BAAe;AAAA,oBACjB,SAAS,GAAG,QAAQ,GAAG,cAAc;AAAA,oBACrC,SAAS;AAAA,sBACP,YAAY,MAAM;AAAA,sBAClB,WAAW;AAAA,sBACX,MAAM,MAAM,SACR,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,WAAW,YAAY,IACjE;AAAA,oBACN;AAAA,kBACF,CAAC;AAAA,kBACD;AAAA,gBACF;AAAA,cACF,OAAO;AACL,qBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAAA,cAC5E;AACA,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,gBAAI,iBAAiB;AACnB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AAAA,YACnF;AACA,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,kBAAI,iBAAiB;AACnB,qBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AAAA,cACnF;AACA,kBAAI;AACF,sBAAM,QAAQ,iBAAiB;AAAA,kBAC7B;AAAA,gBACF,CAAC;AAAA,cACH,UAAE;AACA,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,aAAa,KAAK,IAAI,IAAI;AAC9D,gBAAI,KAAK,gBAAiB;AAC1B,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,MAAM;AACR,oBAAM,QAAQ,kBAAkB,EAAE,KAAK,CAAC;AAAA,YAC1C;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AAtgCnB;AAugCI,QAAI,SAAS,SAAU,MAAK,kBAAkB;AAE9C,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA1gCb,YAAAA;AA0gCgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,iBAAiB;AACnB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F,WAAW,CAAC,cAAc;AACxB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,2BACJ,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,SAAS,CAAC,CAAC,KAAK;AACpE,QAAI,0BAA0B;AAC5B,WAAI,cAAS,kBAAT,mBAAwB,aAAa;AAIvC,YAAI,CAAC,KAAK,sBAAsB;AAC9B,eAAK,yBAAyB;AAAA,QAChC;AAEA,iBAAS,gBAAgB;AAAA,UACvB,GAAG,SAAS;AAAA,UACZ,aAAa;AAAA,QACf;AAEA,cAAM,KAAK,SAAS;AACpB,cAAM,mBACJ,CAAC,EAAC,yBAAI,eACN,yBAAI,wBAAuB,SAC3B,yBAAI,uBAAsB,SAC1B,yBAAI,uBAAsB,SAC1B,yBAAI,iBAAgB;AACtB,YAAI,CAAC,kBAAkB;AACrB,mBAAS,gBAAgB;AACzB,cAAI,iBAAiB;AACnB,iBAAK,QAAQ,MAAM,+BAA+B;AAAA,UACpD;AAAA,QACF;AAAA,MACF;AAGA,UAAI,KAAK,gBAAgB,QAAQ,GAAG;AAClC,aAAK,mBAAmB;AACxB,YAAI,iBAAiB;AACnB,eAAK,QAAQ,MAAM,4BAA2B,UAAK,sBAAL,mBAAwB,UAAU,EAAE;AAAA,QACpF;AAAA,MACF;AAAA,IACF;AACA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AApnC7B;AAqnCI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAhpC/F;AAipCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,0BAAmC,OACnC,KACM;AACN,UAAM,SAAS,OAAO,KAAK;AAC3B,QAAI,CAAC,UAAU,OAAO,OAAO;AAC3B;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,YAAY;AAIlB,QAAI,UAAU,oBAAoB;AAChC,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,UAAU;AAAA,QAClB,YAAY,UAAU;AAAA,QACtB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,UAAU;AAAA,QACnB,IAAI,UAAU;AAAA,MAChB,CAAC;AAAA,IACH;AAEA,QAAI,UAAU,YAAY;AACxB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,UAAU;AAAA,QACnB,IAAI,UAAU;AAAA,MAChB,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,gBAAU,YAAY,MAAM,EAAE;AAAA,IAChC;AAEA,cAAU,YAAY,MAAM;AAC5B,cAAU,aAAa,MAAM;AAC7B,QAAI,CAAC,yBAAyB;AAC5B,gBAAU,gBAAgB,MAAM;AAAA,IAClC;AACA,cAAU,eAAe,MAAM;AAC/B,cAAU,QAAQ;AAAA,EACpB;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,gBAAgB,KAAK;AAAA,MACrB,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,UAAM,cAAc,KAAK;AACzB,UAAM,iCAAiC,eAAe,CAAC,YAAY,gBAAgB;AAGnF,QAAI,eAAe,gCAAgC;AACjD,kBAAY,gBAAgB,MAAM;AAAA,IACpC;AAEA,QAAI,eAAe,CAAC,YAAY,OAAO;AACrC,UAAI,gCAAgC;AAClC,aAAK,gCAAgC;AAAA,MACvC,OAAO;AACL,aAAK,QAAQ,KAAK,uEAAuE;AACzF,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,MACf;AAAA,IACF;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,UAAM,gBAAgB,KAAK;AAE3B,QAAI,cAAc,aAAa,CAAC,eAAe;AAC7C,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AAEnC,YAAI,KAAK,SAAS;AAChB;AAAA,QACF;AAEA,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QACE,CAAC,iBACD,cAAc,uBACd,cAAc,oBAAoB,MAClC;AACA,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,eAAe,CAAC,KAAK,sBAAsB;AAC3D,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,gBAAgB,CAAC,KAAK,wBAAwB;AAC9D,UAAI,KAAK,+BAA+B;AACtC,aAAK,0BAA0B,OAAO,KAAK,6BAA6B;AACxE,aAAK,gCAAgC;AAAA,MACvC,OAAO;AACL,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAIA,QACE,KAAK,2BACJ,cAAc,gBAAgB,cAAc,qBAC7C;AACA,WAAK,yBAAyB;AAAA,IAChC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,SAAK,kBAAkB;AAEvB,QAAI,IAAI,gBAAgB,QAAQ;AAC9B,WAAK,QAAQ,KAAK,2DAA2D;AAC7E;AAAA,IACF;AAEA,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,CAAC,GAAG,MAAM;AACZ,aAAK,QAAQ,KAAK,+CAA+C;AACjE;AAAA,MACF;AACA,UAAI,gBAAgB;AAAA,QAClB,kBAAI,aAAa,OAAO;AAAA,UACtB,QAAQ,GAAG,UAAM,yBAAU,WAAW;AAAA,UACtC,MAAM,GAAG;AAAA,UACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,QAC5C,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,gBAAgB,MAAM;AAAA,EAC5B;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAC;AAAA,EAErB,MAAM,aAAa;AAAA,EAAC;AAAA,EAEpB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,gBAAgB,UAAmC;AACzD,QAAI,KAAK,wBAAwB;AAC/B,aAAO;AAAA,IACT;AACA,QAAI,SAAS,UAAU;AACrB,aAAO;AAAA,IACT;AAEA,UAAM,gBAAgB,SAAS;AAC/B,WACE,CAAC,CAAC,kBACD,cAAc,aACb,cAAc,uBAAuB,QACrC,cAAc,sBAAsB,QACpC,cAAc,sBAAsB,QACpC,cAAc,gBAAgB;AAAA,EAEpC;AACF;","names":["_a"]}
1
+ {"version":3,"sources":["../../../src/beta/realtime/realtime_api.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Session } from '@google/genai';\nimport * as types from '@google/genai';\nimport {\n ActivityHandling,\n type AudioTranscriptionConfig,\n type ContextWindowCompressionConfig,\n GoogleGenAI,\n type HttpOptions,\n Modality,\n type RealtimeInputConfig,\n} from '@google/genai';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Event,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n llm,\n log,\n normalizeLanguage,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';\nimport { type LLMTools } from '../../tools.js';\nimport { toFunctionDeclarations } from '../../utils.js';\nimport type * as api_proto from './api_proto.js';\nimport type { LiveAPIModels, Voice } from './api_proto.js';\n\n// Input audio constants (matching Python)\nconst INPUT_AUDIO_SAMPLE_RATE = 16000;\nconst INPUT_AUDIO_CHANNELS = 1;\n\n// Output audio constants (matching Python)\nconst OUTPUT_AUDIO_SAMPLE_RATE = 24000;\nconst OUTPUT_AUDIO_CHANNELS = 1;\n\nconst LK_GOOGLE_DEBUG = Number(process.env.LK_GOOGLE_DEBUG ?? 0);\n\n// WebSocket close codes (RFC 6455)\nconst WS_CLOSE_NORMAL = 1000;\n/**\n * Default image encoding options for Google Realtime API\n */\nexport const DEFAULT_IMAGE_ENCODE_OPTIONS = {\n format: 'JPEG' as const,\n quality: 75,\n resizeOptions: {\n width: 1024,\n height: 1024,\n strategy: 'scale_aspect_fit' as const,\n },\n};\n\n/**\n * Input transcription result\n */\nexport interface InputTranscription {\n itemId: string;\n transcript: string;\n}\n\n/**\n * Helper function to check if two sets are equal\n */\nfunction setsEqual<T>(a: Set<T>, b: Set<T>): boolean {\n return a.size === b.size && [...a].every((x) => b.has(x));\n}\n\n/**\n * Internal realtime options for Google Realtime API\n */\ninterface RealtimeOptions {\n model: LiveAPIModels | string;\n apiKey?: string;\n voice: Voice | string;\n language?: string;\n responseModalities: Modality[];\n vertexai: boolean;\n project?: string;\n location?: string;\n candidateCount: number;\n temperature?: number;\n maxOutputTokens?: number;\n topP?: number;\n topK?: number;\n presencePenalty?: number;\n frequencyPenalty?: number;\n instructions?: string;\n inputAudioTranscription?: AudioTranscriptionConfig;\n outputAudioTranscription?: AudioTranscriptionConfig;\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n connOptions: APIConnectOptions;\n httpOptions?: HttpOptions;\n enableAffectiveDialog?: boolean;\n proactivity?: boolean;\n realtimeInputConfig?: RealtimeInputConfig;\n contextWindowCompression?: ContextWindowCompressionConfig;\n apiVersion?: string;\n geminiTools?: LLMTools;\n thinkingConfig?: types.ThinkingConfig;\n}\n\n/**\n * Response generation tracking\n */\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel<llm.MessageGeneration>;\n functionChannel: stream.StreamChannel<llm.FunctionCall>;\n\n inputId: string;\n responseId: string;\n textChannel: stream.StreamChannel<string>;\n audioChannel: stream.StreamChannel<AudioFrame>;\n\n inputTranscription: string;\n outputText: string;\n\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n /** @internal */\n _completedTimestamp?: number;\n /** @internal */\n _done: boolean;\n}\n\n/**\n * Google Realtime Model for real-time voice conversations with Gemini models\n */\nexport class RealtimeModel extends llm.RealtimeModel {\n /** @internal */\n _options: RealtimeOptions;\n\n get model(): string {\n return this._options.model;\n }\n\n constructor(\n options: {\n /**\n * Initial system instructions for the model\n */\n instructions?: string;\n\n /**\n * The name of the model to use\n */\n model?: LiveAPIModels | string;\n\n /**\n * Google Gemini API key. If not provided, will attempt to read from GOOGLE_API_KEY environment variable\n */\n apiKey?: string;\n\n /**\n * Voice setting for audio outputs\n */\n voice?: Voice | string;\n\n /**\n * The language (BCP-47 Code) to use for the API\n * See https://ai.google.dev/gemini-api/docs/live#supported-languages\n */\n language?: string;\n\n /**\n * Modalities to use, such as [Modality.TEXT, Modality.AUDIO]\n */\n modalities?: Modality[];\n\n /**\n * Whether to use VertexAI for the API\n */\n vertexai?: boolean;\n\n /**\n * The project ID to use for the API (for VertexAI)\n */\n project?: string;\n\n /**\n * The location to use for the API (for VertexAI)\n */\n location?: string;\n\n /**\n * The number of candidate responses to generate\n */\n candidateCount?: number;\n\n /**\n * Sampling temperature for response generation\n */\n temperature?: number;\n\n /**\n * Maximum number of tokens in the response\n */\n maxOutputTokens?: number;\n\n /**\n * The top-p value for response generation\n */\n topP?: number;\n\n /**\n * The top-k value for response generation\n */\n topK?: number;\n\n /**\n * The presence penalty for response generation\n */\n presencePenalty?: number;\n\n /**\n * The frequency penalty for response generation\n */\n frequencyPenalty?: number;\n\n /**\n * The configuration for input audio transcription\n */\n inputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for output audio transcription\n */\n outputAudioTranscription?: AudioTranscriptionConfig | null;\n\n /**\n * The configuration for image encoding\n */\n imageEncodeOptions?: typeof DEFAULT_IMAGE_ENCODE_OPTIONS;\n\n /**\n * Whether to enable affective dialog\n */\n enableAffectiveDialog?: boolean;\n\n /**\n * Whether to enable proactive audio\n */\n proactivity?: boolean;\n\n /**\n * The configuration for realtime input\n */\n realtimeInputConfig?: RealtimeInputConfig;\n\n /**\n * The configuration for context window compression\n */\n contextWindowCompression?: ContextWindowCompressionConfig;\n\n /**\n * API version to use\n */\n apiVersion?: string;\n\n /**\n * The configuration for the API connection\n */\n connOptions?: APIConnectOptions;\n\n /**\n * HTTP options for API requests\n */\n httpOptions?: HttpOptions;\n\n /**\n * Gemini-specific tools to use for the session\n */\n geminiTools?: LLMTools;\n\n /**\n * Thinking configuration for native audio models.\n * If not set, the model's default thinking behavior is used.\n * Use `\\{ thinkingBudget: 0 \\}` to disable thinking.\n * Use `\\{ thinkingBudget: -1 \\}` for automatic/dynamic thinking.\n */\n thinkingConfig?: types.ThinkingConfig;\n } = {},\n ) {\n const inputAudioTranscription =\n options.inputAudioTranscription === undefined ? {} : options.inputAudioTranscription;\n const outputAudioTranscription =\n options.outputAudioTranscription === undefined ? {} : options.outputAudioTranscription;\n\n let serverTurnDetection = true;\n if (options.realtimeInputConfig?.automaticActivityDetection?.disabled) {\n serverTurnDetection = false;\n }\n\n super({\n messageTruncation: false,\n turnDetection: serverTurnDetection,\n userTranscription: inputAudioTranscription !== null,\n autoToolReplyGeneration: true,\n audioOutput: options.modalities?.includes(Modality.AUDIO) ?? true,\n manualFunctionCalls: false,\n });\n\n // Environment variable fallbacks\n const apiKey = options.apiKey || process.env.GOOGLE_API_KEY;\n const project = options.project || process.env.GOOGLE_CLOUD_PROJECT;\n const location = options.location || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';\n const vertexai = options.vertexai ?? false;\n\n // Model selection based on API type\n const defaultModel = vertexai\n ? 'gemini-live-2.5-flash-native-audio'\n : 'gemini-2.5-flash-native-audio-preview-12-2025';\n\n this._options = {\n model: options.model || defaultModel,\n apiKey,\n voice: options.voice || 'Puck',\n language: options.language ? normalizeLanguage(options.language) : undefined,\n responseModalities: options.modalities || [Modality.AUDIO],\n vertexai,\n project,\n location,\n candidateCount: options.candidateCount || 1,\n temperature: options.temperature,\n maxOutputTokens: options.maxOutputTokens,\n topP: options.topP,\n topK: options.topK,\n presencePenalty: options.presencePenalty,\n frequencyPenalty: options.frequencyPenalty,\n instructions: options.instructions,\n inputAudioTranscription: inputAudioTranscription || undefined,\n outputAudioTranscription: outputAudioTranscription || undefined,\n imageEncodeOptions: options.imageEncodeOptions || DEFAULT_IMAGE_ENCODE_OPTIONS,\n connOptions: options.connOptions || DEFAULT_API_CONNECT_OPTIONS,\n httpOptions: options.httpOptions,\n enableAffectiveDialog: options.enableAffectiveDialog,\n proactivity: options.proactivity,\n realtimeInputConfig: options.realtimeInputConfig,\n contextWindowCompression: options.contextWindowCompression,\n apiVersion: options.apiVersion,\n geminiTools: options.geminiTools,\n thinkingConfig: options.thinkingConfig,\n };\n }\n\n /**\n * Create a new realtime session\n */\n session() {\n return new RealtimeSession(this);\n }\n\n /**\n * Update model options\n */\n updateOptions(options: { voice?: Voice | string; temperature?: number }): void {\n if (options.voice !== undefined) {\n this._options.voice = options.voice;\n }\n if (options.temperature !== undefined) {\n this._options.temperature = options.temperature;\n }\n\n // TODO: Notify active sessions of option changes\n }\n\n /**\n * Close the model and cleanup resources\n */\n async close(): Promise<void> {\n // TODO: Implementation depends on session management\n }\n}\n\n/**\n * Google Realtime Session for real-time voice conversations\n *\n * This session provides real-time streaming capabilities with Google's Gemini models,\n * supporting both text and audio modalities with function calling capabilities.\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private _chatCtx = llm.ChatContext.empty();\n\n private options: RealtimeOptions;\n private geminiDeclarations: types.FunctionDeclaration[] = [];\n private messageChannel = new Queue<api_proto.ClientEvents>();\n private inputResampler?: AudioResampler;\n private inputResamplerInputRate?: number;\n private instructions?: string;\n private currentGeneration?: ResponseGeneration;\n private bstream: AudioByteStream;\n\n // Google-specific properties\n private activeSession?: Session;\n private sessionShouldClose = new Event();\n private responseCreatedFutures: { [id: string]: Future<llm.GenerationCreatedEvent> } = {};\n private pendingGenerationFut?: Future<llm.GenerationCreatedEvent>;\n\n private sessionResumptionHandle?: string;\n private inUserActivity = false;\n private sessionLock = new Mutex();\n private numRetries = 0;\n private hasReceivedAudioInput = false;\n private pendingInterruptText = false;\n private earlyCompletionPending = false;\n private pendingToolCallIds = new Set<string>();\n private generationPendingTurnComplete?: ResponseGeneration;\n\n #client: GoogleGenAI;\n #task: Promise<void>;\n #logger = log();\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.options = realtimeModel._options;\n this.bstream = new AudioByteStream(\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n INPUT_AUDIO_SAMPLE_RATE / 20,\n ); // 50ms chunks\n\n const { apiKey, project, location, vertexai, enableAffectiveDialog, proactivity } =\n this.options;\n\n const apiVersion =\n !this.options.apiVersion && (enableAffectiveDialog || proactivity)\n ? 'v1alpha'\n : this.options.apiVersion;\n\n const httpOptions = {\n ...this.options.httpOptions,\n apiVersion,\n timeout: this.options.connOptions.timeoutMs,\n };\n\n const clientOptions: types.GoogleGenAIOptions = vertexai\n ? {\n vertexai: true,\n project,\n location,\n httpOptions,\n }\n : {\n apiKey,\n httpOptions,\n };\n\n this.#client = new GoogleGenAI(clientOptions);\n this.#task = this.#mainTask();\n }\n\n private async closeActiveSession(): Promise<void> {\n const unlock = await this.sessionLock.lock();\n\n if (this.activeSession) {\n try {\n await this.activeSession.close();\n } catch (error) {\n this.#logger.warn({ error }, 'Error closing Gemini session');\n } finally {\n this.activeSession = undefined;\n }\n }\n this.earlyCompletionPending = false;\n this.pendingInterruptText = false;\n\n this.pendingToolCallIds.clear();\n if (this.generationPendingTurnComplete) {\n this.markCurrentGenerationDone(false, this.generationPendingTurnComplete);\n this.generationPendingTurnComplete = undefined;\n }\n unlock();\n }\n\n private markRestartNeeded(): void {\n if (!this.sessionShouldClose.isSet) {\n this.sessionShouldClose.set();\n this.messageChannel = new Queue();\n }\n }\n\n private getToolResultsForRealtime(\n ctx: llm.ChatContext,\n vertexai: boolean,\n ): types.LiveClientToolResponse | undefined {\n const toolResponses: types.FunctionResponse[] = [];\n\n for (const item of ctx.items) {\n if (item.type === 'function_call_output') {\n const response: types.FunctionResponse = {\n id: item.callId,\n name: item.name,\n response: { output: item.output },\n };\n\n if (!vertexai) {\n response.id = item.callId;\n }\n\n toolResponses.push(response);\n }\n }\n\n return toolResponses.length > 0 ? { functionResponses: toolResponses } : undefined;\n }\n\n updateOptions(options: {\n voice?: Voice | string;\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n }) {\n let shouldRestart = false;\n\n if (options.voice !== undefined && this.options.voice !== options.voice) {\n this.options.voice = options.voice;\n shouldRestart = true;\n }\n\n if (options.temperature !== undefined && this.options.temperature !== options.temperature) {\n this.options.temperature = options.temperature;\n shouldRestart = true;\n }\n\n if (shouldRestart) {\n this.markRestartNeeded();\n }\n }\n\n async updateInstructions(instructions: string): Promise<void> {\n if (this.options.instructions === undefined || this.options.instructions !== instructions) {\n this.options.instructions = instructions;\n this.markRestartNeeded();\n }\n }\n\n async updateChatCtx(chatCtx: llm.ChatContext): Promise<void> {\n const unlock = await this.sessionLock.lock();\n try {\n if (!this.activeSession) {\n this._chatCtx = chatCtx.copy();\n return;\n }\n } finally {\n unlock();\n }\n\n const diffOps = llm.computeChatCtxDiff(this._chatCtx, chatCtx);\n\n if (diffOps.toRemove.length > 0) {\n this.#logger.warn('Gemini Live does not support removing messages');\n }\n\n const appendCtx = llm.ChatContext.empty();\n for (const [, itemId] of diffOps.toCreate) {\n const item = chatCtx.getById(itemId);\n if (item) {\n appendCtx.items.push(item);\n }\n }\n\n if (appendCtx.items.length > 0) {\n const [turns] = await appendCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n const toolResults = this.getToolResultsForRealtime(appendCtx, this.options.vertexai);\n\n if (turns.length > 0) {\n const shouldSendRealtimeText = this.pendingInterruptText;\n\n if (shouldSendRealtimeText) {\n for (const turn of turns as types.Content[]) {\n if (turn.role !== 'user') continue;\n // Realtime text drives live activity/interrupts\n // { type: content: turnComplete: true } alone does not reliably preempt a streaming response in Gemini Live.\n const text = (turn.parts || [])\n .map((part) => (part as { text?: string }).text)\n .filter((value): value is string => !!value)\n .join('');\n if (text) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: { text },\n });\n this.pendingInterruptText = false;\n }\n }\n }\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns: turns as types.Content[],\n turnComplete: false,\n },\n });\n }\n\n if (toolResults) {\n this.sendClientEvent({\n type: 'tool_response',\n value: toolResults,\n });\n }\n }\n\n // since we don't have a view of the history on the server side, we'll assume\n // the current state is accurate. this isn't perfect because removals aren't done.\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: llm.ToolContext): Promise<void> {\n const newDeclarations = toFunctionDeclarations(tools);\n const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));\n const newToolNames = new Set(newDeclarations.map((f) => f.name));\n\n if (!setsEqual(currentToolNames, newToolNames)) {\n this.geminiDeclarations = newDeclarations;\n this._tools = tools;\n this.markRestartNeeded();\n }\n }\n\n get chatCtx(): llm.ChatContext {\n return this._chatCtx.copy();\n }\n\n get tools(): llm.ToolContext {\n return { ...this._tools };\n }\n\n get manualActivityDetection(): boolean {\n return this.options.realtimeInputConfig?.automaticActivityDetection?.disabled ?? false;\n }\n\n pushAudio(frame: AudioFrame): void {\n if (this.pendingToolCallIds.size > 0) return;\n\n // Track that we've received audio input\n this.hasReceivedAudioInput = true;\n\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n const realtimeInput: types.LiveClientRealtimeInput = {\n mediaChunks: [\n {\n mimeType: 'audio/pcm',\n data: Buffer.from(nf.data.buffer).toString('base64'),\n },\n ],\n };\n this.sendClientEvent({\n type: 'realtime_input',\n value: realtimeInput,\n });\n }\n }\n }\n\n pushVideo(_: VideoFrame): void {\n // TODO(brian): implement push video frames\n }\n\n private sendClientEvent(event: api_proto.ClientEvents) {\n this.messageChannel.put(event);\n }\n\n async generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent> {\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.#logger.warn(\n 'generateReply called while another generation is pending, cancelling previous.',\n );\n this.pendingGenerationFut.reject(new Error('Superseded by new generate_reply call'));\n }\n\n const fut = new Future<llm.GenerationCreatedEvent>();\n this.pendingGenerationFut = fut;\n\n if (this.inUserActivity) {\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityEnd: {},\n },\n });\n this.inUserActivity = false;\n }\n\n // Gemini requires the last message to end with user's turn\n // so we need to add a placeholder user turn in order to trigger a new generation\n const turns: types.Content[] = [];\n if (instructions !== undefined) {\n turns.push({\n parts: [{ text: instructions }],\n role: 'model',\n });\n }\n turns.push({\n parts: [{ text: '.' }],\n role: 'user',\n });\n\n this.sendClientEvent({\n type: 'content',\n value: {\n turns,\n turnComplete: true,\n },\n });\n\n const timeoutHandle = setTimeout(() => {\n if (!fut.done) {\n fut.reject(new Error('generateReply timed out waiting for generation_created event.'));\n if (this.pendingGenerationFut === fut) {\n this.pendingGenerationFut = undefined;\n }\n }\n }, 5000);\n\n fut.await.finally(() => clearTimeout(timeoutHandle));\n\n return fut.await;\n }\n\n startUserActivity(): void {\n if (!this.manualActivityDetection) {\n return;\n }\n\n if (this.pendingToolCallIds.size > 0) return;\n\n if (!this.inUserActivity) {\n this.inUserActivity = true;\n this.sendClientEvent({\n type: 'realtime_input',\n value: {\n activityStart: {},\n },\n });\n }\n }\n\n private generationHasOutput(gen: ResponseGeneration): boolean {\n return Boolean(gen.outputText) || gen._firstTokenTimestamp !== undefined;\n }\n\n async interrupt() {\n // Gemini Live treats activity start as interruption, so we rely on startUserActivity to handle it\n if (this.options.realtimeInputConfig?.activityHandling === ActivityHandling.NO_INTERRUPTION) {\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug('interrupt skipped (activityHandling = NO_INTERRUPTION)');\n }\n return;\n }\n if (this.currentGeneration && !this.currentGeneration._done) {\n this.pendingInterruptText = true;\n if (this.generationHasOutput(this.currentGeneration)) {\n this.earlyCompletionPending = true;\n this.markCurrentGenerationDone();\n }\n }\n this.startUserActivity();\n }\n\n async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {\n this.#logger.warn('truncate is not supported by the Google Realtime API.');\n }\n\n async close(): Promise<void> {\n super.close();\n this.#closed = true;\n\n this.sessionShouldClose.set();\n\n await this.closeActiveSession();\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n this.pendingGenerationFut.reject(new Error('Session closed'));\n }\n\n for (const fut of Object.values(this.responseCreatedFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed before response created'));\n }\n }\n this.responseCreatedFutures = {};\n\n if (this.currentGeneration) {\n this.markCurrentGenerationDone();\n }\n }\n\n async #mainTask(): Promise<void> {\n const maxRetries = this.options.connOptions.maxRetry;\n\n while (!this.#closed) {\n // previous session might not be closed yet, we'll do it here.\n await this.closeActiveSession();\n\n this.sessionShouldClose.clear();\n const config = this.buildConnectConfig();\n\n try {\n this.#logger.debug('Connecting to Gemini Realtime API...');\n\n const sessionOpened = new Event();\n const session = await this.#client.live.connect({\n model: this.options.model,\n callbacks: {\n onopen: () => sessionOpened.set(),\n onmessage: (message: types.LiveServerMessage) => {\n this.onReceiveMessage(session, message);\n },\n // onerror is called for network-level errors (connection refused, DNS failure, TLS errors).\n // Application-level errors (e.g., invalid model name) come through onclose with error codes.\n onerror: (error: ErrorEvent) => {\n this.#logger.error('Gemini Live session error:', error);\n if (!this.sessionShouldClose.isSet) {\n this.markRestartNeeded();\n }\n },\n onclose: (event: CloseEvent) => {\n // Surface WebSocket close errors to the user instead of silently swallowing them\n if (event.code !== WS_CLOSE_NORMAL) {\n // Note: WebSocket close reasons are limited to 123 bytes by RFC 6455,\n // so Google's error messages may be truncated at the protocol level\n const isTruncated = event.reason && event.reason.length >= 120;\n const truncationNote = isTruncated\n ? ' (message may be truncated - check model name and API permissions)'\n : '';\n const errorMsg = event.reason || `WebSocket closed with code ${event.code}`;\n this.#logger.error(`Gemini Live session error: ${errorMsg}${truncationNote}`);\n\n this.emitError(\n new APIStatusError({\n message: `${errorMsg}${truncationNote}`,\n options: {\n statusCode: event.code,\n retryable: false,\n body: event.reason\n ? { reason: event.reason, code: event.code, truncated: isTruncated }\n : null,\n },\n }),\n false,\n );\n } else {\n this.#logger.debug('Gemini Live session closed:', event.code, event.reason);\n }\n this.markCurrentGenerationDone();\n },\n },\n config,\n });\n\n await sessionOpened.wait();\n\n const unlock = await this.sessionLock.lock();\n try {\n this.activeSession = session;\n\n // Send existing chat context\n const [turns] = await this._chatCtx\n .copy({\n excludeFunctionCall: true,\n })\n .toProviderFormat('google', false);\n\n if (turns.length > 0) {\n await session.sendClientContent({\n turns,\n turnComplete: false,\n });\n }\n } finally {\n unlock();\n }\n\n const sendTask = Task.from((controller) => this.sendTask(session, controller));\n const restartWaitTask = Task.from(({ signal }) => {\n const abortEvent = new Event();\n signal.addEventListener('abort', () => abortEvent.set());\n return Promise.race([this.sessionShouldClose.wait(), abortEvent.wait()]);\n });\n\n await Promise.race([sendTask.result, restartWaitTask.result]);\n\n // TODO(brian): handle error from tasks\n\n if (!restartWaitTask.done && this.#closed) {\n break;\n }\n\n await cancelAndWait([sendTask, restartWaitTask], 2000);\n } catch (error) {\n this.#logger.error(`Gemini Realtime API error: ${error}`);\n\n if (this.#closed) break;\n\n if (maxRetries === 0) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: 'Failed to connect to Gemini Live',\n });\n }\n\n if (this.numRetries >= maxRetries) {\n this.emitError(error as Error, false);\n throw new APIConnectionError({\n message: `Failed to connect to Gemini Live after ${maxRetries} attempts`,\n });\n }\n\n const retryInterval =\n this.numRetries === 100 ? 0 : this.options.connOptions.retryIntervalMs;\n\n this.#logger.warn(\n {\n attempt: this.numRetries,\n maxRetries,\n },\n `Gemini Realtime API connection failed, retrying in ${retryInterval}ms`,\n );\n\n await delay(retryInterval);\n this.numRetries++;\n } finally {\n await this.closeActiveSession();\n }\n }\n }\n\n private async sendTask(session: types.Session, controller: AbortController): Promise<void> {\n try {\n while (!this.#closed && !this.sessionShouldClose.isSet && !controller.signal.aborted) {\n const msg = await this.messageChannel.get();\n if (controller.signal.aborted) break;\n\n const unlock = await this.sessionLock.lock();\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n break;\n }\n } finally {\n unlock();\n }\n\n switch (msg.type) {\n case 'content':\n const { turns, turnComplete } = msg.value;\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n }\n await session.sendClientContent({\n turns,\n turnComplete: turnComplete ?? true,\n });\n break;\n case 'tool_response':\n const { functionResponses } = msg.value;\n if (functionResponses) {\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`(client) -> ${JSON.stringify(this.loggableClientEvent(msg))}`);\n }\n try {\n await session.sendToolResponse({\n functionResponses,\n });\n } finally {\n for (const fr of functionResponses) {\n if (fr?.id) this.pendingToolCallIds.delete(fr.id);\n }\n }\n }\n break;\n case 'realtime_input':\n const { mediaChunks, activityStart, activityEnd, text } = msg.value;\n if (this.pendingToolCallIds.size > 0) break;\n if (mediaChunks) {\n for (const mediaChunk of mediaChunks) {\n await session.sendRealtimeInput({ media: mediaChunk });\n }\n }\n if (text) {\n await session.sendRealtimeInput({ text });\n }\n if (activityStart) await session.sendRealtimeInput({ activityStart });\n if (activityEnd) await session.sendRealtimeInput({ activityEnd });\n break;\n default:\n this.#logger.warn(`Warning: Received unhandled message type: ${msg.type}`);\n break;\n }\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in send task: ${e}`);\n this.markRestartNeeded();\n }\n } finally {\n this.#logger.debug(\n {\n closed: this.#closed,\n sessionShouldClose: this.sessionShouldClose.isSet,\n aborted: controller.signal.aborted,\n },\n 'send task finished.',\n );\n }\n }\n\n private async onReceiveMessage(\n session: types.Session,\n response: types.LiveServerMessage,\n ): Promise<void> {\n // Skip logging verbose audio data events\n const hasAudioData = response.serverContent?.modelTurn?.parts?.some(\n (part) => part.inlineData?.data,\n );\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n } else if (!hasAudioData) {\n this.#logger.debug(`(server) <- ${JSON.stringify(this.loggableServerMessage(response))}`);\n }\n const unlock = await this.sessionLock.lock();\n\n try {\n if (this.sessionShouldClose.isSet || this.activeSession !== session) {\n this.#logger.debug('onReceiveMessage: Session changed or closed, stopping receive.');\n return;\n }\n } finally {\n unlock();\n }\n\n const shouldStartNewGeneration =\n !this.currentGeneration || this.currentGeneration._done || !!this.pendingGenerationFut;\n if (shouldStartNewGeneration) {\n if (response.serverContent?.interrupted) {\n // Two cases when an interrupted event is sent without an active generation:\n // 1) generation done but playout not finished (turnComplete -> interrupted)\n // 2) generation not started (interrupted -> turnComplete)\n if (!this.pendingGenerationFut) {\n this.handleInputSpeechStarted();\n }\n\n response.serverContent = {\n ...response.serverContent,\n interrupted: undefined,\n };\n\n const sc = response.serverContent;\n const hasServerContent =\n !!sc?.modelTurn ||\n sc?.outputTranscription != null ||\n sc?.inputTranscription != null ||\n sc?.generationComplete != null ||\n sc?.turnComplete != null;\n if (!hasServerContent) {\n response.serverContent = undefined;\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug('ignoring empty server content');\n }\n }\n }\n\n // start new generation for serverContent or for standalone toolCalls\n if (this.isNewGeneration(response)) {\n this.startNewGeneration();\n if (LK_GOOGLE_DEBUG) {\n this.#logger.debug(`new generation started: ${this.currentGeneration?.responseId}`);\n }\n }\n }\n if (response.sessionResumptionUpdate) {\n if (\n response.sessionResumptionUpdate.resumable &&\n response.sessionResumptionUpdate.newHandle\n ) {\n this.sessionResumptionHandle = response.sessionResumptionUpdate.newHandle;\n }\n }\n\n try {\n if (response.serverContent) {\n this.handleServerContent(response.serverContent);\n }\n\n if (response.toolCall) {\n this.handleToolCall(response.toolCall);\n }\n\n if (response.toolCallCancellation) {\n this.handleToolCallCancellation(response.toolCallCancellation);\n }\n\n if (response.usageMetadata) {\n this.handleUsageMetadata(response.usageMetadata);\n }\n\n if (response.goAway) {\n this.handleGoAway(response.goAway);\n }\n\n if (this.numRetries > 0) {\n this.numRetries = 0;\n }\n } catch (e) {\n if (!this.sessionShouldClose.isSet) {\n this.#logger.error(`Error in onReceiveMessage: ${e}`);\n this.markRestartNeeded();\n }\n }\n }\n\n /// Truncate large base64/audio payloads for logging to avoid flooding logs\n private truncateString(data: string, maxLength: number = 30): string {\n return data.length > maxLength ? `${data.slice(0, maxLength)}…` : data;\n }\n\n private loggableClientEvent(\n event: api_proto.ClientEvents,\n maxLength: number = 30,\n ): Record<string, unknown> {\n const obj: any = { ...event };\n if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {\n obj.value = {\n ...obj.value,\n mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(\n (mc) => ({\n ...mc,\n data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,\n }),\n ),\n };\n }\n return obj;\n }\n\n private loggableServerMessage(\n message: types.LiveServerMessage,\n maxLength: number = 30,\n ): Record<string, unknown> {\n const obj: any = { ...message };\n if (\n obj.serverContent &&\n obj.serverContent.modelTurn &&\n Array.isArray(obj.serverContent.modelTurn.parts)\n ) {\n obj.serverContent = { ...obj.serverContent };\n obj.serverContent.modelTurn = { ...obj.serverContent.modelTurn };\n obj.serverContent.modelTurn.parts = obj.serverContent.modelTurn.parts.map((part: any) => {\n if (part?.inlineData?.data && typeof part.inlineData.data === 'string') {\n return {\n ...part,\n inlineData: {\n ...part.inlineData,\n data: this.truncateString(part.inlineData.data, maxLength),\n },\n };\n }\n return part;\n });\n }\n return obj;\n }\n\n private markCurrentGenerationDone(\n keepFunctionChannelOpen: boolean = false,\n gen?: ResponseGeneration,\n ): void {\n const target = gen ?? this.currentGeneration;\n if (!target || target._done) {\n return;\n }\n\n this.handleInputSpeechStopped();\n\n const targetGen = target;\n\n // The only way we'd know that the transcription is complete is by when they are\n // done with generation\n if (targetGen.inputTranscription) {\n this.emit('input_audio_transcription_completed', {\n itemId: targetGen.inputId,\n transcript: targetGen.inputTranscription,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n\n // since gemini doesn't give us a view of the chat history on the server side,\n // we would handle it manually here\n this._chatCtx.addMessage({\n role: 'user',\n content: targetGen.inputTranscription,\n id: targetGen.inputId,\n });\n }\n\n if (targetGen.outputText) {\n this._chatCtx.addMessage({\n role: 'assistant',\n content: targetGen.outputText,\n id: targetGen.responseId,\n });\n }\n\n if (this.options.outputAudioTranscription === undefined) {\n // close the text data of transcription synchronizer\n targetGen.textChannel.write('');\n }\n\n targetGen.textChannel.close();\n targetGen.audioChannel.close();\n if (!keepFunctionChannelOpen) {\n targetGen.functionChannel.close();\n }\n targetGen.messageChannel.close();\n targetGen._done = true;\n }\n\n private emitError(error: Error, recoverable: boolean): void {\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label to realtime model\n label: 'google_realtime',\n error,\n recoverable,\n });\n }\n\n private buildConnectConfig(): types.LiveConnectConfig {\n const opts = this.options;\n\n const config: types.LiveConnectConfig = {\n thinkingConfig: opts.thinkingConfig,\n responseModalities: opts.responseModalities,\n systemInstruction: opts.instructions\n ? {\n parts: [{ text: opts.instructions }],\n }\n : undefined,\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: opts.voice as Voice,\n },\n },\n languageCode: opts.language,\n },\n tools: [\n {\n functionDeclarations: this.geminiDeclarations,\n ...this.options.geminiTools,\n },\n ],\n inputAudioTranscription: opts.inputAudioTranscription,\n outputAudioTranscription: opts.outputAudioTranscription,\n sessionResumption: {\n handle: this.sessionResumptionHandle,\n },\n };\n\n // Add generation fields at TOP LEVEL (NO generationConfig!)\n if (opts.temperature !== undefined) {\n config.temperature = opts.temperature;\n }\n if (opts.maxOutputTokens !== undefined) {\n config.maxOutputTokens = opts.maxOutputTokens;\n }\n if (opts.topP !== undefined) {\n config.topP = opts.topP;\n }\n if (opts.topK !== undefined) {\n config.topK = opts.topK;\n }\n\n if (opts.proactivity !== undefined) {\n config.proactivity = { proactiveAudio: opts.proactivity };\n }\n\n if (opts.enableAffectiveDialog !== undefined) {\n config.enableAffectiveDialog = opts.enableAffectiveDialog;\n }\n\n if (opts.realtimeInputConfig !== undefined) {\n config.realtimeInputConfig = opts.realtimeInputConfig;\n }\n\n if (opts.contextWindowCompression !== undefined) {\n config.contextWindowCompression = opts.contextWindowCompression;\n }\n\n return config;\n }\n\n private startNewGeneration(): void {\n const previousGen = this.currentGeneration;\n const previousHadOpenFunctionChannel = previousGen && !previousGen.functionChannel.closed;\n\n // close functionChannel of previous generation if still open (no toolCall arrived)\n if (previousGen && previousHadOpenFunctionChannel) {\n previousGen.functionChannel.close();\n }\n\n if (previousGen && !previousGen._done) {\n if (previousHadOpenFunctionChannel) {\n this.generationPendingTurnComplete = previousGen;\n } else {\n this.#logger.warn('Starting new generation while another is active. Finalizing previous.');\n this.markCurrentGenerationDone();\n }\n }\n\n const responseId = shortuuid('GR_');\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel<llm.MessageGeneration>(),\n functionChannel: stream.createStreamChannel<llm.FunctionCall>(),\n responseId,\n inputId: shortuuid('GI_'),\n textChannel: stream.createStreamChannel<string>(),\n audioChannel: stream.createStreamChannel<AudioFrame>(),\n inputTranscription: '',\n outputText: '',\n _createdTimestamp: Date.now(),\n _done: false,\n };\n\n // Close audio stream if audio output is not supported by the model\n if (!this._realtimeModel.capabilities.audioOutput) {\n this.currentGeneration.audioChannel.close();\n }\n\n // Determine modalities based on the model's audio_output capability\n const modalities: ('text' | 'audio')[] = this._realtimeModel.capabilities.audioOutput\n ? ['audio', 'text']\n : ['text'];\n\n this.currentGeneration.messageChannel.write({\n messageId: responseId,\n textStream: this.currentGeneration.textChannel.stream(),\n audioStream: this.currentGeneration.audioChannel.stream(),\n modalities: Promise.resolve(modalities),\n });\n\n const generationEvent: llm.GenerationCreatedEvent = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n responseId,\n };\n\n if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {\n generationEvent.userInitiated = true;\n this.pendingGenerationFut.resolve(generationEvent);\n this.pendingGenerationFut = undefined;\n } else {\n // emit input_speech_started event before starting an agent initiated generation\n // to interrupt the previous audio playout if any\n this.handleInputSpeechStarted();\n }\n\n this.emit('generation_created', generationEvent);\n }\n\n private handleInputSpeechStarted(): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputSpeechStopped(): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: false,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleServerContent(serverContent: types.LiveServerContent): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received server content but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n const discardOutput = this.earlyCompletionPending;\n\n if (serverContent.modelTurn && !discardOutput) {\n const turn = serverContent.modelTurn;\n\n for (const part of turn.parts || []) {\n // bypass reasoning/thought output\n if (part.thought) {\n continue;\n }\n\n if (part.text) {\n gen.outputText += part.text;\n gen.textChannel.write(part.text);\n }\n\n if (part.inlineData) {\n if (!gen._firstTokenTimestamp) {\n gen._firstTokenTimestamp = Date.now();\n }\n\n try {\n if (!part.inlineData.data) {\n throw new Error('frameData is not bytes');\n }\n\n const binaryString = atob(part.inlineData.data);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n const int16Array = new Int16Array(bytes.buffer);\n const audioFrame = new AudioFrame(\n int16Array,\n OUTPUT_AUDIO_SAMPLE_RATE,\n OUTPUT_AUDIO_CHANNELS,\n int16Array.length / OUTPUT_AUDIO_CHANNELS,\n );\n\n gen.audioChannel.write(audioFrame);\n } catch (error) {\n this.#logger.error('Error processing audio data:', error);\n }\n }\n }\n }\n\n if (serverContent.inputTranscription && serverContent.inputTranscription.text) {\n let text = serverContent.inputTranscription.text;\n\n if (gen.inputTranscription === '') {\n text = text.trimStart();\n }\n\n gen.inputTranscription += text;\n this.emit('input_audio_transcription_completed', {\n itemId: gen.inputId,\n transcript: gen.inputTranscription,\n isFinal: false,\n } as llm.InputTranscriptionCompleted);\n }\n\n if (\n !discardOutput &&\n serverContent.outputTranscription &&\n serverContent.outputTranscription.text\n ) {\n const text = serverContent.outputTranscription.text;\n gen.outputText += text;\n gen.textChannel.write(text);\n }\n\n if (serverContent.generationComplete || serverContent.turnComplete) {\n gen._completedTimestamp = Date.now();\n }\n\n if (serverContent.interrupted && !this.pendingGenerationFut) {\n this.handleInputSpeechStarted();\n }\n\n if (serverContent.turnComplete && !this.earlyCompletionPending) {\n if (this.generationPendingTurnComplete) {\n this.markCurrentGenerationDone(false, this.generationPendingTurnComplete);\n this.generationPendingTurnComplete = undefined;\n } else {\n this.markCurrentGenerationDone();\n }\n }\n\n // Assume Gemini emits turnComplete/generationComplete before any new generation content.\n // We keep discarding until that signal to avoid old stream spillover after interrupts.\n if (\n this.earlyCompletionPending &&\n (serverContent.turnComplete || serverContent.generationComplete)\n ) {\n this.earlyCompletionPending = false;\n }\n }\n\n private handleToolCall(toolCall: types.LiveServerToolCall): void {\n if (!this.currentGeneration) {\n this.#logger.warn('received tool call but no active generation.');\n return;\n }\n\n const gen = this.currentGeneration;\n\n if (gen.functionChannel.closed) {\n this.#logger.warn('received tool call but functionChannel is already closed.');\n return;\n }\n\n for (const fc of toolCall.functionCalls || []) {\n if (!fc.name) {\n this.#logger.warn('received function call without name, skipping');\n continue;\n }\n const callId = fc.id || shortuuid('fnc-call-');\n this.pendingToolCallIds.add(callId);\n gen.functionChannel.write(\n llm.FunctionCall.create({\n callId,\n name: fc.name,\n args: fc.args ? JSON.stringify(fc.args) : '',\n }),\n );\n }\n }\n\n private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {\n this.#logger.warn(\n {\n functionCallIds: cancellation.ids,\n },\n 'server cancelled tool calls',\n );\n for (const id of cancellation.ids || []) {\n this.pendingToolCallIds.delete(id);\n }\n }\n\n private handleUsageMetadata(usage: types.UsageMetadata): void {\n if (!this.currentGeneration) {\n this.#logger.debug('Received usage metadata but no active generation');\n return;\n }\n\n const gen = this.currentGeneration;\n const createdTimestamp = gen._createdTimestamp;\n const firstTokenTimestamp = gen._firstTokenTimestamp;\n const completedTimestamp = gen._completedTimestamp || Date.now();\n\n // Calculate metrics\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = completedTimestamp - createdTimestamp;\n\n const inputTokens = usage.promptTokenCount || 0;\n const outputTokens = usage.responseTokenCount || 0;\n const totalTokens = usage.totalTokenCount || 0;\n\n const realtimeMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: gen.responseId,\n ttftMs,\n durationMs,\n cancelled: gen._done && !gen._completedTimestamp,\n label: 'google_realtime',\n inputTokens,\n outputTokens,\n totalTokens,\n tokensPerSecond: durationMs > 0 ? outputTokens / (durationMs / 1000) : 0,\n inputTokenDetails: {\n ...this.tokenDetailsMap(usage.promptTokensDetails),\n cachedTokens: (usage.cacheTokensDetails || []).reduce(\n (sum, detail) => sum + (detail.tokenCount || 0),\n 0,\n ),\n cachedTokensDetails: this.tokenDetailsMap(usage.cacheTokensDetails),\n },\n outputTokenDetails: this.tokenDetailsMap(usage.responseTokensDetails),\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n }\n\n private tokenDetailsMap(tokenDetails: types.ModalityTokenCount[] | undefined): {\n audioTokens: number;\n textTokens: number;\n imageTokens: number;\n } {\n const tokenDetailsMap = { audioTokens: 0, textTokens: 0, imageTokens: 0 };\n if (!tokenDetails) {\n return tokenDetailsMap;\n }\n\n for (const tokenDetail of tokenDetails) {\n if (!tokenDetail.tokenCount) {\n continue;\n }\n\n if (tokenDetail.modality === types.MediaModality.AUDIO) {\n tokenDetailsMap.audioTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.TEXT) {\n tokenDetailsMap.textTokens += tokenDetail.tokenCount;\n } else if (tokenDetail.modality === types.MediaModality.IMAGE) {\n tokenDetailsMap.imageTokens += tokenDetail.tokenCount;\n }\n }\n return tokenDetailsMap;\n }\n\n private handleGoAway(goAway: types.LiveServerGoAway): void {\n this.#logger.warn({ timeLeft: goAway.timeLeft }, 'Gemini server indicates disconnection soon.');\n // TODO(brian): this isn't a seamless reconnection just yet\n this.sessionShouldClose.set();\n }\n\n async commitAudio() {}\n\n async clearAudio() {}\n\n private *resampleAudio(frame: AudioFrame): Generator<AudioFrame> {\n if (this.inputResampler) {\n if (frame.sampleRate !== this.inputResamplerInputRate) {\n // input audio changed to a different sample rate\n this.inputResampler = undefined;\n this.inputResamplerInputRate = undefined;\n }\n }\n\n if (\n this.inputResampler === undefined &&\n (frame.sampleRate !== INPUT_AUDIO_SAMPLE_RATE || frame.channels !== INPUT_AUDIO_CHANNELS)\n ) {\n this.inputResampler = new AudioResampler(\n frame.sampleRate,\n INPUT_AUDIO_SAMPLE_RATE,\n INPUT_AUDIO_CHANNELS,\n );\n this.inputResamplerInputRate = frame.sampleRate;\n }\n\n if (this.inputResampler) {\n // TODO(brian): flush the resampler when the input source is changed\n for (const resampledFrame of this.inputResampler.push(frame)) {\n yield resampledFrame;\n }\n } else {\n yield frame;\n }\n }\n\n private isNewGeneration(response: types.LiveServerMessage) {\n if (this.earlyCompletionPending) {\n return false;\n }\n if (response.toolCall) {\n return true;\n }\n\n const serverContent = response.serverContent;\n return (\n !!serverContent &&\n (serverContent.modelTurn ||\n serverContent.outputTranscription != null ||\n serverContent.inputTranscription != null ||\n serverContent.generationComplete != null ||\n serverContent.turnComplete != null)\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,YAAuB;AACvB,mBAQO;AAEP,oBAgBO;AACP,mBAAsB;AACtB,sBAA4D;AAC5D,mBAA8B;AAC9B,mBAAuC;AAKvC,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAG7B,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAE9B,MAAM,kBAAkB,OAAO,QAAQ,IAAI,mBAAmB,CAAC;AAG/D,MAAM,kBAAkB;AAIjB,MAAM,+BAA+B;AAAA,EAC1C,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,eAAe;AAAA,IACb,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAaA,SAAS,UAAa,GAAW,GAAoB;AACnD,SAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC1D;AAgEO,MAAM,sBAAsB,kBAAI,cAAc;AAAA;AAAA,EAEnD;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,YACE,UAgJI,CAAC,GACL;AAvSJ;AAwSI,UAAM,0BACJ,QAAQ,4BAA4B,SAAY,CAAC,IAAI,QAAQ;AAC/D,UAAM,2BACJ,QAAQ,6BAA6B,SAAY,CAAC,IAAI,QAAQ;AAEhE,QAAI,sBAAsB;AAC1B,SAAI,mBAAQ,wBAAR,mBAA6B,+BAA7B,mBAAyD,UAAU;AACrE,4BAAsB;AAAA,IACxB;AAEA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe;AAAA,MACf,mBAAmB,4BAA4B;AAAA,MAC/C,yBAAyB;AAAA,MACzB,eAAa,aAAQ,eAAR,mBAAoB,SAAS,sBAAS,WAAU;AAAA,MAC7D,qBAAqB;AAAA,IACvB,CAAC;AAGD,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAC7C,UAAM,UAAU,QAAQ,WAAW,QAAQ,IAAI;AAC/C,UAAM,WAAW,QAAQ,YAAY,QAAQ,IAAI,yBAAyB;AAC1E,UAAM,WAAW,QAAQ,YAAY;AAGrC,UAAM,eAAe,WACjB,uCACA;AAEJ,SAAK,WAAW;AAAA,MACd,OAAO,QAAQ,SAAS;AAAA,MACxB;AAAA,MACA,OAAO,QAAQ,SAAS;AAAA,MACxB,UAAU,QAAQ,eAAW,iCAAkB,QAAQ,QAAQ,IAAI;AAAA,MACnE,oBAAoB,QAAQ,cAAc,CAAC,sBAAS,KAAK;AAAA,MACzD;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,QAAQ,kBAAkB;AAAA,MAC1C,aAAa,QAAQ;AAAA,MACrB,iBAAiB,QAAQ;AAAA,MACzB,MAAM,QAAQ;AAAA,MACd,MAAM,QAAQ;AAAA,MACd,iBAAiB,QAAQ;AAAA,MACzB,kBAAkB,QAAQ;AAAA,MAC1B,cAAc,QAAQ;AAAA,MACtB,yBAAyB,2BAA2B;AAAA,MACpD,0BAA0B,4BAA4B;AAAA,MACtD,oBAAoB,QAAQ,sBAAsB;AAAA,MAClD,aAAa,QAAQ,eAAe;AAAA,MACpC,aAAa,QAAQ;AAAA,MACrB,uBAAuB,QAAQ;AAAA,MAC/B,aAAa,QAAQ;AAAA,MACrB,qBAAqB,QAAQ;AAAA,MAC7B,0BAA0B,QAAQ;AAAA,MAClC,YAAY,QAAQ;AAAA,MACpB,aAAa,QAAQ;AAAA,MACrB,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,SAAiE;AAC7E,QAAI,QAAQ,UAAU,QAAW;AAC/B,WAAK,SAAS,QAAQ,QAAQ;AAAA,IAChC;AACA,QAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAK,SAAS,cAAc,QAAQ;AAAA,IACtC;AAAA,EAGF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAAA,EAE7B;AACF;AAQO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,WAAW,kBAAI,YAAY,MAAM;AAAA,EAEjC;AAAA,EACA,qBAAkD,CAAC;AAAA,EACnD,iBAAiB,IAAI,oBAA8B;AAAA,EACnD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA,qBAAqB,IAAI,oBAAM;AAAA,EAC/B,yBAA+E,CAAC;AAAA,EAChF;AAAA,EAEA;AAAA,EACA,iBAAiB;AAAA,EACjB,cAAc,IAAI,mBAAM;AAAA,EACxB,aAAa;AAAA,EACb,wBAAwB;AAAA,EACxB,uBAAuB;AAAA,EACvB,yBAAyB;AAAA,EACzB,qBAAqB,oBAAI,IAAY;AAAA,EACrC;AAAA,EAER;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B;AAEA,UAAM,EAAE,QAAQ,SAAS,UAAU,UAAU,uBAAuB,YAAY,IAC9E,KAAK;AAEP,UAAM,aACJ,CAAC,KAAK,QAAQ,eAAe,yBAAyB,eAClD,YACA,KAAK,QAAQ;AAEnB,UAAM,cAAc;AAAA,MAClB,GAAG,KAAK,QAAQ;AAAA,MAChB;AAAA,MACA,SAAS,KAAK,QAAQ,YAAY;AAAA,IACpC;AAEA,UAAM,gBAA0C,WAC5C;AAAA,MACE,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,IACF,IACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAEJ,SAAK,UAAU,IAAI,yBAAY,aAAa;AAC5C,SAAK,QAAQ,KAAK,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAc,qBAAoC;AAChD,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI,KAAK,eAAe;AACtB,UAAI;AACF,cAAM,KAAK,cAAc,MAAM;AAAA,MACjC,SAAS,OAAO;AACd,aAAK,QAAQ,KAAK,EAAE,MAAM,GAAG,8BAA8B;AAAA,MAC7D,UAAE;AACA,aAAK,gBAAgB;AAAA,MACvB;AAAA,IACF;AACA,SAAK,yBAAyB;AAC9B,SAAK,uBAAuB;AAE5B,SAAK,mBAAmB,MAAM;AAC9B,QAAI,KAAK,+BAA+B;AACtC,WAAK,0BAA0B,OAAO,KAAK,6BAA6B;AACxE,WAAK,gCAAgC;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAA0B;AAChC,QAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,WAAK,mBAAmB,IAAI;AAC5B,WAAK,iBAAiB,IAAI,oBAAM;AAAA,IAClC;AAAA,EACF;AAAA,EAEQ,0BACN,KACA,UAC0C;AAC1C,UAAM,gBAA0C,CAAC;AAEjD,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,KAAK,SAAS,wBAAwB;AACxC,cAAM,WAAmC;AAAA,UACvC,IAAI,KAAK;AAAA,UACT,MAAM,KAAK;AAAA,UACX,UAAU,EAAE,QAAQ,KAAK,OAAO;AAAA,QAClC;AAEA,YAAI,CAAC,UAAU;AACb,mBAAS,KAAK,KAAK;AAAA,QACrB;AAEA,sBAAc,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF;AAEA,WAAO,cAAc,SAAS,IAAI,EAAE,mBAAmB,cAAc,IAAI;AAAA,EAC3E;AAAA,EAEA,cAAc,SAIX;AACD,QAAI,gBAAgB;AAEpB,QAAI,QAAQ,UAAU,UAAa,KAAK,QAAQ,UAAU,QAAQ,OAAO;AACvE,WAAK,QAAQ,QAAQ,QAAQ;AAC7B,sBAAgB;AAAA,IAClB;AAEA,QAAI,QAAQ,gBAAgB,UAAa,KAAK,QAAQ,gBAAgB,QAAQ,aAAa;AACzF,WAAK,QAAQ,cAAc,QAAQ;AACnC,sBAAgB;AAAA,IAClB;AAEA,QAAI,eAAe;AACjB,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,cAAqC;AAC5D,QAAI,KAAK,QAAQ,iBAAiB,UAAa,KAAK,QAAQ,iBAAiB,cAAc;AACzF,WAAK,QAAQ,eAAe;AAC5B,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,SAAyC;AAC3D,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,QAAI;AACF,UAAI,CAAC,KAAK,eAAe;AACvB,aAAK,WAAW,QAAQ,KAAK;AAC7B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,kBAAI,mBAAmB,KAAK,UAAU,OAAO;AAE7D,QAAI,QAAQ,SAAS,SAAS,GAAG;AAC/B,WAAK,QAAQ,KAAK,gDAAgD;AAAA,IACpE;AAEA,UAAM,YAAY,kBAAI,YAAY,MAAM;AACxC,eAAW,CAAC,EAAE,MAAM,KAAK,QAAQ,UAAU;AACzC,YAAM,OAAO,QAAQ,QAAQ,MAAM;AACnC,UAAI,MAAM;AACR,kBAAU,MAAM,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,UAAU,MAAM,SAAS,GAAG;AAC9B,YAAM,CAAC,KAAK,IAAI,MAAM,UACnB,KAAK;AAAA,QACJ,qBAAqB;AAAA,MACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,YAAM,cAAc,KAAK,0BAA0B,WAAW,KAAK,QAAQ,QAAQ;AAEnF,UAAI,MAAM,SAAS,GAAG;AACpB,cAAM,yBAAyB,KAAK;AAEpC,YAAI,wBAAwB;AAC1B,qBAAW,QAAQ,OAA0B;AAC3C,gBAAI,KAAK,SAAS,OAAQ;AAG1B,kBAAM,QAAQ,KAAK,SAAS,CAAC,GAC1B,IAAI,CAAC,SAAU,KAA2B,IAAI,EAC9C,OAAO,CAAC,UAA2B,CAAC,CAAC,KAAK,EAC1C,KAAK,EAAE;AACV,gBAAI,MAAM;AACR,mBAAK,gBAAgB;AAAA,gBACnB,MAAM;AAAA,gBACN,OAAO,EAAE,KAAK;AAAA,cAChB,CAAC;AACD,mBAAK,uBAAuB;AAAA,YAC9B;AAAA,UACF;AAAA,QACF;AAEA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,YACL;AAAA,YACA,cAAc;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,aAAa;AACf,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAIA,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAuC;AACvD,UAAM,sBAAkB,qCAAuB,KAAK;AACpD,UAAM,mBAAmB,IAAI,IAAI,KAAK,mBAAmB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAC3E,UAAM,eAAe,IAAI,IAAI,gBAAgB,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAE/D,QAAI,CAAC,UAAU,kBAAkB,YAAY,GAAG;AAC9C,WAAK,qBAAqB;AAC1B,WAAK,SAAS;AACd,WAAK,kBAAkB;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,IAAI,UAA2B;AAC7B,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,QAAyB;AAC3B,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,0BAAmC;AAzoBzC;AA0oBI,aAAO,gBAAK,QAAQ,wBAAb,mBAAkC,+BAAlC,mBAA8D,aAAY;AAAA,EACnF;AAAA,EAEA,UAAU,OAAyB;AACjC,QAAI,KAAK,mBAAmB,OAAO,EAAG;AAGtC,SAAK,wBAAwB;AAE7B,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,cAAM,gBAA+C;AAAA,UACnD,aAAa;AAAA,YACX;AAAA,cACE,UAAU;AAAA,cACV,MAAM,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACrD;AAAA,UACF;AAAA,QACF;AACA,aAAK,gBAAgB;AAAA,UACnB,MAAM;AAAA,UACN,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,GAAqB;AAAA,EAE/B;AAAA,EAEQ,gBAAgB,OAA+B;AACrD,SAAK,eAAe,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,QAAQ;AAAA,QACX;AAAA,MACF;AACA,WAAK,qBAAqB,OAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,IACrF;AAEA,UAAM,MAAM,IAAI,qBAAmC;AACnD,SAAK,uBAAuB;AAE5B,QAAI,KAAK,gBAAgB;AACvB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,aAAa,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAIA,UAAM,QAAyB,CAAC;AAChC,QAAI,iBAAiB,QAAW;AAC9B,YAAM,KAAK;AAAA,QACT,OAAO,CAAC,EAAE,MAAM,aAAa,CAAC;AAAA,QAC9B,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AACA,UAAM,KAAK;AAAA,MACT,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAED,SAAK,gBAAgB;AAAA,MACnB,MAAM;AAAA,MACN,OAAO;AAAA,QACL;AAAA,QACA,cAAc;AAAA,MAChB;AAAA,IACF,CAAC;AAED,UAAM,gBAAgB,WAAW,MAAM;AACrC,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,+DAA+D,CAAC;AACrF,YAAI,KAAK,yBAAyB,KAAK;AACrC,eAAK,uBAAuB;AAAA,QAC9B;AAAA,MACF;AAAA,IACF,GAAG,GAAI;AAEP,QAAI,MAAM,QAAQ,MAAM,aAAa,aAAa,CAAC;AAEnD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,yBAAyB;AACjC;AAAA,IACF;AAEA,QAAI,KAAK,mBAAmB,OAAO,EAAG;AAEtC,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB;AACtB,WAAK,gBAAgB;AAAA,QACnB,MAAM;AAAA,QACN,OAAO;AAAA,UACL,eAAe,CAAC;AAAA,QAClB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,oBAAoB,KAAkC;AAC5D,WAAO,QAAQ,IAAI,UAAU,KAAK,IAAI,yBAAyB;AAAA,EACjE;AAAA,EAEA,MAAM,YAAY;AA5vBpB;AA8vBI,UAAI,UAAK,QAAQ,wBAAb,mBAAkC,sBAAqB,8BAAiB,iBAAiB;AAC3F,UAAI,iBAAiB;AACnB,aAAK,QAAQ,MAAM,wDAAwD;AAAA,MAC7E;AACA;AAAA,IACF;AACA,QAAI,KAAK,qBAAqB,CAAC,KAAK,kBAAkB,OAAO;AAC3D,WAAK,uBAAuB;AAC5B,UAAI,KAAK,oBAAoB,KAAK,iBAAiB,GAAG;AACpD,aAAK,yBAAyB;AAC9B,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AACA,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,UAA+E;AAC5F,SAAK,QAAQ,KAAK,uDAAuD;AAAA,EAC3E;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,MAAM;AACZ,SAAK,UAAU;AAEf,SAAK,mBAAmB,IAAI;AAE5B,UAAM,KAAK,mBAAmB;AAE9B,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,WAAK,qBAAqB,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,IAC9D;AAEA,eAAW,OAAO,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC5D,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,wCAAwC,CAAC;AAAA,MAChE;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,QAAI,KAAK,mBAAmB;AAC1B,WAAK,0BAA0B;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,YAA2B;AAC/B,UAAM,aAAa,KAAK,QAAQ,YAAY;AAE5C,WAAO,CAAC,KAAK,SAAS;AAEpB,YAAM,KAAK,mBAAmB;AAE9B,WAAK,mBAAmB,MAAM;AAC9B,YAAM,SAAS,KAAK,mBAAmB;AAEvC,UAAI;AACF,aAAK,QAAQ,MAAM,sCAAsC;AAEzD,cAAM,gBAAgB,IAAI,oBAAM;AAChC,cAAM,UAAU,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,UAC9C,OAAO,KAAK,QAAQ;AAAA,UACpB,WAAW;AAAA,YACT,QAAQ,MAAM,cAAc,IAAI;AAAA,YAChC,WAAW,CAAC,YAAqC;AAC/C,mBAAK,iBAAiB,SAAS,OAAO;AAAA,YACxC;AAAA;AAAA;AAAA,YAGA,SAAS,CAAC,UAAsB;AAC9B,mBAAK,QAAQ,MAAM,8BAA8B,KAAK;AACtD,kBAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,qBAAK,kBAAkB;AAAA,cACzB;AAAA,YACF;AAAA,YACA,SAAS,CAAC,UAAsB;AAE9B,kBAAI,MAAM,SAAS,iBAAiB;AAGlC,sBAAM,cAAc,MAAM,UAAU,MAAM,OAAO,UAAU;AAC3D,sBAAM,iBAAiB,cACnB,uEACA;AACJ,sBAAM,WAAW,MAAM,UAAU,8BAA8B,MAAM,IAAI;AACzE,qBAAK,QAAQ,MAAM,8BAA8B,QAAQ,GAAG,cAAc,EAAE;AAE5E,qBAAK;AAAA,kBACH,IAAI,6BAAe;AAAA,oBACjB,SAAS,GAAG,QAAQ,GAAG,cAAc;AAAA,oBACrC,SAAS;AAAA,sBACP,YAAY,MAAM;AAAA,sBAClB,WAAW;AAAA,sBACX,MAAM,MAAM,SACR,EAAE,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,WAAW,YAAY,IACjE;AAAA,oBACN;AAAA,kBACF,CAAC;AAAA,kBACD;AAAA,gBACF;AAAA,cACF,OAAO;AACL,qBAAK,QAAQ,MAAM,+BAA+B,MAAM,MAAM,MAAM,MAAM;AAAA,cAC5E;AACA,mBAAK,0BAA0B;AAAA,YACjC;AAAA,UACF;AAAA,UACA;AAAA,QACF,CAAC;AAED,cAAM,cAAc,KAAK;AAEzB,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,eAAK,gBAAgB;AAGrB,gBAAM,CAAC,KAAK,IAAI,MAAM,KAAK,SACxB,KAAK;AAAA,YACJ,qBAAqB;AAAA,UACvB,CAAC,EACA,iBAAiB,UAAU,KAAK;AAEnC,cAAI,MAAM,SAAS,GAAG;AACpB,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc;AAAA,YAChB,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,cAAM,WAAW,mBAAK,KAAK,CAAC,eAAe,KAAK,SAAS,SAAS,UAAU,CAAC;AAC7E,cAAM,kBAAkB,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AAChD,gBAAM,aAAa,IAAI,oBAAM;AAC7B,iBAAO,iBAAiB,SAAS,MAAM,WAAW,IAAI,CAAC;AACvD,iBAAO,QAAQ,KAAK,CAAC,KAAK,mBAAmB,KAAK,GAAG,WAAW,KAAK,CAAC,CAAC;AAAA,QACzE,CAAC;AAED,cAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC;AAI5D,YAAI,CAAC,gBAAgB,QAAQ,KAAK,SAAS;AACzC;AAAA,QACF;AAEA,kBAAM,6BAAc,CAAC,UAAU,eAAe,GAAG,GAAI;AAAA,MACvD,SAAS,OAAO;AACd,aAAK,QAAQ,MAAM,8BAA8B,KAAK,EAAE;AAExD,YAAI,KAAK,QAAS;AAElB,YAAI,eAAe,GAAG;AACpB,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAEA,YAAI,KAAK,cAAc,YAAY;AACjC,eAAK,UAAU,OAAgB,KAAK;AACpC,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,0CAA0C,UAAU;AAAA,UAC/D,CAAC;AAAA,QACH;AAEA,cAAM,gBACJ,KAAK,eAAe,MAAM,IAAI,KAAK,QAAQ,YAAY;AAEzD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS,KAAK;AAAA,YACd;AAAA,UACF;AAAA,UACA,sDAAsD,aAAa;AAAA,QACrE;AAEA,kBAAM,qBAAM,aAAa;AACzB,aAAK;AAAA,MACP,UAAE;AACA,cAAM,KAAK,mBAAmB;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,SAAS,SAAwB,YAA4C;AACzF,QAAI;AACF,aAAO,CAAC,KAAK,WAAW,CAAC,KAAK,mBAAmB,SAAS,CAAC,WAAW,OAAO,SAAS;AACpF,cAAM,MAAM,MAAM,KAAK,eAAe,IAAI;AAC1C,YAAI,WAAW,OAAO,QAAS;AAE/B,cAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAC3C,YAAI;AACF,cAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO;AAAA,QACT;AAEA,gBAAQ,IAAI,MAAM;AAAA,UAChB,KAAK;AACH,kBAAM,EAAE,OAAO,aAAa,IAAI,IAAI;AACpC,gBAAI,iBAAiB;AACnB,mBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AAAA,YACnF;AACA,kBAAM,QAAQ,kBAAkB;AAAA,cAC9B;AAAA,cACA,cAAc,gBAAgB;AAAA,YAChC,CAAC;AACD;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,kBAAkB,IAAI,IAAI;AAClC,gBAAI,mBAAmB;AACrB,kBAAI,iBAAiB;AACnB,qBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,oBAAoB,GAAG,CAAC,CAAC,EAAE;AAAA,cACnF;AACA,kBAAI;AACF,sBAAM,QAAQ,iBAAiB;AAAA,kBAC7B;AAAA,gBACF,CAAC;AAAA,cACH,UAAE;AACA,2BAAW,MAAM,mBAAmB;AAClC,sBAAI,yBAAI,GAAI,MAAK,mBAAmB,OAAO,GAAG,EAAE;AAAA,gBAClD;AAAA,cACF;AAAA,YACF;AACA;AAAA,UACF,KAAK;AACH,kBAAM,EAAE,aAAa,eAAe,aAAa,KAAK,IAAI,IAAI;AAC9D,gBAAI,KAAK,mBAAmB,OAAO,EAAG;AACtC,gBAAI,aAAa;AACf,yBAAW,cAAc,aAAa;AACpC,sBAAM,QAAQ,kBAAkB,EAAE,OAAO,WAAW,CAAC;AAAA,cACvD;AAAA,YACF;AACA,gBAAI,MAAM;AACR,oBAAM,QAAQ,kBAAkB,EAAE,KAAK,CAAC;AAAA,YAC1C;AACA,gBAAI,cAAe,OAAM,QAAQ,kBAAkB,EAAE,cAAc,CAAC;AACpE,gBAAI,YAAa,OAAM,QAAQ,kBAAkB,EAAE,YAAY,CAAC;AAChE;AAAA,UACF;AACE,iBAAK,QAAQ,KAAK,6CAA6C,IAAI,IAAI,EAAE;AACzE;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,uBAAuB,CAAC,EAAE;AAC7C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,oBAAoB,KAAK,mBAAmB;AAAA,UAC5C,SAAS,WAAW,OAAO;AAAA,QAC7B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,iBACZ,SACA,UACe;AAxgCnB;AA0gCI,UAAM,gBAAe,0BAAS,kBAAT,mBAAwB,cAAxB,mBAAmC,UAAnC,mBAA0C;AAAA,MAC7D,CAAC,SAAM;AA3gCb,YAAAA;AA2gCgB,gBAAAA,MAAA,KAAK,eAAL,gBAAAA,IAAiB;AAAA;AAAA;AAE7B,QAAI,iBAAiB;AACnB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F,WAAW,CAAC,cAAc;AACxB,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,sBAAsB,QAAQ,CAAC,CAAC,EAAE;AAAA,IAC1F;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,KAAK;AAE3C,QAAI;AACF,UAAI,KAAK,mBAAmB,SAAS,KAAK,kBAAkB,SAAS;AACnE,aAAK,QAAQ,MAAM,gEAAgE;AACnF;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO;AAAA,IACT;AAEA,UAAM,2BACJ,CAAC,KAAK,qBAAqB,KAAK,kBAAkB,SAAS,CAAC,CAAC,KAAK;AACpE,QAAI,0BAA0B;AAC5B,WAAI,cAAS,kBAAT,mBAAwB,aAAa;AAIvC,YAAI,CAAC,KAAK,sBAAsB;AAC9B,eAAK,yBAAyB;AAAA,QAChC;AAEA,iBAAS,gBAAgB;AAAA,UACvB,GAAG,SAAS;AAAA,UACZ,aAAa;AAAA,QACf;AAEA,cAAM,KAAK,SAAS;AACpB,cAAM,mBACJ,CAAC,EAAC,yBAAI,eACN,yBAAI,wBAAuB,SAC3B,yBAAI,uBAAsB,SAC1B,yBAAI,uBAAsB,SAC1B,yBAAI,iBAAgB;AACtB,YAAI,CAAC,kBAAkB;AACrB,mBAAS,gBAAgB;AACzB,cAAI,iBAAiB;AACnB,iBAAK,QAAQ,MAAM,+BAA+B;AAAA,UACpD;AAAA,QACF;AAAA,MACF;AAGA,UAAI,KAAK,gBAAgB,QAAQ,GAAG;AAClC,aAAK,mBAAmB;AACxB,YAAI,iBAAiB;AACnB,eAAK,QAAQ,MAAM,4BAA2B,UAAK,sBAAL,mBAAwB,UAAU,EAAE;AAAA,QACpF;AAAA,MACF;AAAA,IACF;AACA,QAAI,SAAS,yBAAyB;AACpC,UACE,SAAS,wBAAwB,aACjC,SAAS,wBAAwB,WACjC;AACA,aAAK,0BAA0B,SAAS,wBAAwB;AAAA,MAClE;AAAA,IACF;AAEA,QAAI;AACF,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,UAAU;AACrB,aAAK,eAAe,SAAS,QAAQ;AAAA,MACvC;AAEA,UAAI,SAAS,sBAAsB;AACjC,aAAK,2BAA2B,SAAS,oBAAoB;AAAA,MAC/D;AAEA,UAAI,SAAS,eAAe;AAC1B,aAAK,oBAAoB,SAAS,aAAa;AAAA,MACjD;AAEA,UAAI,SAAS,QAAQ;AACnB,aAAK,aAAa,SAAS,MAAM;AAAA,MACnC;AAEA,UAAI,KAAK,aAAa,GAAG;AACvB,aAAK,aAAa;AAAA,MACpB;AAAA,IACF,SAAS,GAAG;AACV,UAAI,CAAC,KAAK,mBAAmB,OAAO;AAClC,aAAK,QAAQ,MAAM,8BAA8B,CAAC,EAAE;AACpD,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGQ,eAAe,MAAc,YAAoB,IAAY;AACnE,WAAO,KAAK,SAAS,YAAY,GAAG,KAAK,MAAM,GAAG,SAAS,CAAC,WAAM;AAAA,EACpE;AAAA,EAEQ,oBACN,OACA,YAAoB,IACK;AArnC7B;AAsnCI,UAAM,MAAW,EAAE,GAAG,MAAM;AAC5B,QAAI,IAAI,SAAS,sBAAoB,SAAI,UAAJ,mBAAW,cAAa;AAC3D,UAAI,QAAQ;AAAA,QACV,GAAG,IAAI;AAAA,QACP,aAAc,IAAI,MAAM,YAA4D;AAAA,UAClF,CAAC,QAAQ;AAAA,YACP,GAAG;AAAA,YACH,MAAM,OAAO,GAAG,SAAS,WAAW,KAAK,eAAe,GAAG,MAAM,SAAS,IAAI,GAAG;AAAA,UACnF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,sBACN,SACA,YAAoB,IACK;AACzB,UAAM,MAAW,EAAE,GAAG,QAAQ;AAC9B,QACE,IAAI,iBACJ,IAAI,cAAc,aAClB,MAAM,QAAQ,IAAI,cAAc,UAAU,KAAK,GAC/C;AACA,UAAI,gBAAgB,EAAE,GAAG,IAAI,cAAc;AAC3C,UAAI,cAAc,YAAY,EAAE,GAAG,IAAI,cAAc,UAAU;AAC/D,UAAI,cAAc,UAAU,QAAQ,IAAI,cAAc,UAAU,MAAM,IAAI,CAAC,SAAc;AAjpC/F;AAkpCQ,cAAI,kCAAM,eAAN,mBAAkB,SAAQ,OAAO,KAAK,WAAW,SAAS,UAAU;AACtE,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,YAAY;AAAA,cACV,GAAG,KAAK;AAAA,cACR,MAAM,KAAK,eAAe,KAAK,WAAW,MAAM,SAAS;AAAA,YAC3D;AAAA,UACF;AAAA,QACF;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,0BACN,0BAAmC,OACnC,KACM;AACN,UAAM,SAAS,OAAO,KAAK;AAC3B,QAAI,CAAC,UAAU,OAAO,OAAO;AAC3B;AAAA,IACF;AAEA,SAAK,yBAAyB;AAE9B,UAAM,YAAY;AAIlB,QAAI,UAAU,oBAAoB;AAChC,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,UAAU;AAAA,QAClB,YAAY,UAAU;AAAA,QACtB,SAAS;AAAA,MACX,CAAoC;AAIpC,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,UAAU;AAAA,QACnB,IAAI,UAAU;AAAA,MAChB,CAAC;AAAA,IACH;AAEA,QAAI,UAAU,YAAY;AACxB,WAAK,SAAS,WAAW;AAAA,QACvB,MAAM;AAAA,QACN,SAAS,UAAU;AAAA,QACnB,IAAI,UAAU;AAAA,MAChB,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,6BAA6B,QAAW;AAEvD,gBAAU,YAAY,MAAM,EAAE;AAAA,IAChC;AAEA,cAAU,YAAY,MAAM;AAC5B,cAAU,aAAa,MAAM;AAC7B,QAAI,CAAC,yBAAyB;AAC5B,gBAAU,gBAAgB,MAAM;AAAA,IAClC;AACA,cAAU,eAAe,MAAM;AAC/B,cAAU,QAAQ;AAAA,EACpB;AAAA,EAEQ,UAAU,OAAc,aAA4B;AAC1D,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,qBAA8C;AACpD,UAAM,OAAO,KAAK;AAElB,UAAM,SAAkC;AAAA,MACtC,gBAAgB,KAAK;AAAA,MACrB,oBAAoB,KAAK;AAAA,MACzB,mBAAmB,KAAK,eACpB;AAAA,QACE,OAAO,CAAC,EAAE,MAAM,KAAK,aAAa,CAAC;AAAA,MACrC,IACA;AAAA,MACJ,cAAc;AAAA,QACZ,aAAa;AAAA,UACX,qBAAqB;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,QACF;AAAA,QACA,cAAc,KAAK;AAAA,MACrB;AAAA,MACA,OAAO;AAAA,QACL;AAAA,UACE,sBAAsB,KAAK;AAAA,UAC3B,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,MACA,yBAAyB,KAAK;AAAA,MAC9B,0BAA0B,KAAK;AAAA,MAC/B,mBAAmB;AAAA,QACjB,QAAQ,KAAK;AAAA,MACf;AAAA,IACF;AAGA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,KAAK;AAAA,IAC5B;AACA,QAAI,KAAK,oBAAoB,QAAW;AACtC,aAAO,kBAAkB,KAAK;AAAA,IAChC;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AACA,QAAI,KAAK,SAAS,QAAW;AAC3B,aAAO,OAAO,KAAK;AAAA,IACrB;AAEA,QAAI,KAAK,gBAAgB,QAAW;AAClC,aAAO,cAAc,EAAE,gBAAgB,KAAK,YAAY;AAAA,IAC1D;AAEA,QAAI,KAAK,0BAA0B,QAAW;AAC5C,aAAO,wBAAwB,KAAK;AAAA,IACtC;AAEA,QAAI,KAAK,wBAAwB,QAAW;AAC1C,aAAO,sBAAsB,KAAK;AAAA,IACpC;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,2BAA2B,KAAK;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,qBAA2B;AACjC,UAAM,cAAc,KAAK;AACzB,UAAM,iCAAiC,eAAe,CAAC,YAAY,gBAAgB;AAGnF,QAAI,eAAe,gCAAgC;AACjD,kBAAY,gBAAgB,MAAM;AAAA,IACpC;AAEA,QAAI,eAAe,CAAC,YAAY,OAAO;AACrC,UAAI,gCAAgC;AAClC,aAAK,gCAAgC;AAAA,MACvC,OAAO;AACL,aAAK,QAAQ,KAAK,uEAAuE;AACzF,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,UAAM,iBAAa,yBAAU,KAAK;AAClC,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D;AAAA,MACA,aAAS,yBAAU,KAAK;AAAA,MACxB,aAAa,qBAAO,oBAA4B;AAAA,MAChD,cAAc,qBAAO,oBAAgC;AAAA,MACrD,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,mBAAmB,KAAK,IAAI;AAAA,MAC5B,OAAO;AAAA,IACT;AAGA,QAAI,CAAC,KAAK,eAAe,aAAa,aAAa;AACjD,WAAK,kBAAkB,aAAa,MAAM;AAAA,IAC5C;AAGA,UAAM,aAAmC,KAAK,eAAe,aAAa,cACtE,CAAC,SAAS,MAAM,IAChB,CAAC,MAAM;AAEX,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,KAAK,kBAAkB,YAAY,OAAO;AAAA,MACtD,aAAa,KAAK,kBAAkB,aAAa,OAAO;AAAA,MACxD,YAAY,QAAQ,QAAQ,UAAU;AAAA,IACxC,CAAC;AAED,UAAM,kBAA8C;AAAA,MAClD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,MACf;AAAA,IACF;AAEA,QAAI,KAAK,wBAAwB,CAAC,KAAK,qBAAqB,MAAM;AAChE,sBAAgB,gBAAgB;AAChC,WAAK,qBAAqB,QAAQ,eAAe;AACjD,WAAK,uBAAuB;AAAA,IAC9B,OAAO;AAGL,WAAK,yBAAyB;AAAA,IAChC;AAEA,SAAK,KAAK,sBAAsB,eAAe;AAAA,EACjD;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,2BAAiC;AACvC,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B;AAAA,IAC5B,CAAgC;AAAA,EAClC;AAAA,EAEQ,oBAAoB,eAA8C;AACxE,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,mDAAmD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,UAAM,gBAAgB,KAAK;AAE3B,QAAI,cAAc,aAAa,CAAC,eAAe;AAC7C,YAAM,OAAO,cAAc;AAE3B,iBAAW,QAAQ,KAAK,SAAS,CAAC,GAAG;AAEnC,YAAI,KAAK,SAAS;AAChB;AAAA,QACF;AAEA,YAAI,KAAK,MAAM;AACb,cAAI,cAAc,KAAK;AACvB,cAAI,YAAY,MAAM,KAAK,IAAI;AAAA,QACjC;AAEA,YAAI,KAAK,YAAY;AACnB,cAAI,CAAC,IAAI,sBAAsB;AAC7B,gBAAI,uBAAuB,KAAK,IAAI;AAAA,UACtC;AAEA,cAAI;AACF,gBAAI,CAAC,KAAK,WAAW,MAAM;AACzB,oBAAM,IAAI,MAAM,wBAAwB;AAAA,YAC1C;AAEA,kBAAM,eAAe,KAAK,KAAK,WAAW,IAAI;AAC9C,kBAAM,MAAM,aAAa;AACzB,kBAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,qBAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,oBAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,YACtC;AAEA,kBAAM,aAAa,IAAI,WAAW,MAAM,MAAM;AAC9C,kBAAM,aAAa,IAAI;AAAA,cACrB;AAAA,cACA;AAAA,cACA;AAAA,cACA,WAAW,SAAS;AAAA,YACtB;AAEA,gBAAI,aAAa,MAAM,UAAU;AAAA,UACnC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,gCAAgC,KAAK;AAAA,UAC1D;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,sBAAsB,cAAc,mBAAmB,MAAM;AAC7E,UAAI,OAAO,cAAc,mBAAmB;AAE5C,UAAI,IAAI,uBAAuB,IAAI;AACjC,eAAO,KAAK,UAAU;AAAA,MACxB;AAEA,UAAI,sBAAsB;AAC1B,WAAK,KAAK,uCAAuC;AAAA,QAC/C,QAAQ,IAAI;AAAA,QACZ,YAAY,IAAI;AAAA,QAChB,SAAS;AAAA,MACX,CAAoC;AAAA,IACtC;AAEA,QACE,CAAC,iBACD,cAAc,uBACd,cAAc,oBAAoB,MAClC;AACA,YAAM,OAAO,cAAc,oBAAoB;AAC/C,UAAI,cAAc;AAClB,UAAI,YAAY,MAAM,IAAI;AAAA,IAC5B;AAEA,QAAI,cAAc,sBAAsB,cAAc,cAAc;AAClE,UAAI,sBAAsB,KAAK,IAAI;AAAA,IACrC;AAEA,QAAI,cAAc,eAAe,CAAC,KAAK,sBAAsB;AAC3D,WAAK,yBAAyB;AAAA,IAChC;AAEA,QAAI,cAAc,gBAAgB,CAAC,KAAK,wBAAwB;AAC9D,UAAI,KAAK,+BAA+B;AACtC,aAAK,0BAA0B,OAAO,KAAK,6BAA6B;AACxE,aAAK,gCAAgC;AAAA,MACvC,OAAO;AACL,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAIA,QACE,KAAK,2BACJ,cAAc,gBAAgB,cAAc,qBAC7C;AACA,WAAK,yBAAyB;AAAA,IAChC;AAAA,EACF;AAAA,EAEQ,eAAe,UAA0C;AAC/D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,KAAK,8CAA8C;AAChE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AAEjB,QAAI,IAAI,gBAAgB,QAAQ;AAC9B,WAAK,QAAQ,KAAK,2DAA2D;AAC7E;AAAA,IACF;AAEA,eAAW,MAAM,SAAS,iBAAiB,CAAC,GAAG;AAC7C,UAAI,CAAC,GAAG,MAAM;AACZ,aAAK,QAAQ,KAAK,+CAA+C;AACjE;AAAA,MACF;AACA,YAAM,SAAS,GAAG,UAAM,yBAAU,WAAW;AAC7C,WAAK,mBAAmB,IAAI,MAAM;AAClC,UAAI,gBAAgB;AAAA,QAClB,kBAAI,aAAa,OAAO;AAAA,UACtB;AAAA,UACA,MAAM,GAAG;AAAA,UACT,MAAM,GAAG,OAAO,KAAK,UAAU,GAAG,IAAI,IAAI;AAAA,QAC5C,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,2BAA2B,cAA0D;AAC3F,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,iBAAiB,aAAa;AAAA,MAChC;AAAA,MACA;AAAA,IACF;AACA,eAAW,MAAM,aAAa,OAAO,CAAC,GAAG;AACvC,WAAK,mBAAmB,OAAO,EAAE;AAAA,IACnC;AAAA,EACF;AAAA,EAEQ,oBAAoB,OAAkC;AAC5D,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,QAAQ,MAAM,kDAAkD;AACrE;AAAA,IACF;AAEA,UAAM,MAAM,KAAK;AACjB,UAAM,mBAAmB,IAAI;AAC7B,UAAM,sBAAsB,IAAI;AAChC,UAAM,qBAAqB,IAAI,uBAAuB,KAAK,IAAI;AAG/D,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,qBAAqB;AAExC,UAAM,cAAc,MAAM,oBAAoB;AAC9C,UAAM,eAAe,MAAM,sBAAsB;AACjD,UAAM,cAAc,MAAM,mBAAmB;AAE7C,UAAM,kBAAkB;AAAA,MACtB,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA,WAAW,IAAI,SAAS,CAAC,IAAI;AAAA,MAC7B,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,aAAa,IAAI,gBAAgB,aAAa,OAAQ;AAAA,MACvE,mBAAmB;AAAA,QACjB,GAAG,KAAK,gBAAgB,MAAM,mBAAmB;AAAA,QACjD,eAAe,MAAM,sBAAsB,CAAC,GAAG;AAAA,UAC7C,CAAC,KAAK,WAAW,OAAO,OAAO,cAAc;AAAA,UAC7C;AAAA,QACF;AAAA,QACA,qBAAqB,KAAK,gBAAgB,MAAM,kBAAkB;AAAA,MACpE;AAAA,MACA,oBAAoB,KAAK,gBAAgB,MAAM,qBAAqB;AAAA,IACtE;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAChD;AAAA,EAEQ,gBAAgB,cAItB;AACA,UAAM,kBAAkB,EAAE,aAAa,GAAG,YAAY,GAAG,aAAa,EAAE;AACxE,QAAI,CAAC,cAAc;AACjB,aAAO;AAAA,IACT;AAEA,eAAW,eAAe,cAAc;AACtC,UAAI,CAAC,YAAY,YAAY;AAC3B;AAAA,MACF;AAEA,UAAI,YAAY,aAAa,MAAM,cAAc,OAAO;AACtD,wBAAgB,eAAe,YAAY;AAAA,MAC7C,WAAW,YAAY,aAAa,MAAM,cAAc,MAAM;AAC5D,wBAAgB,cAAc,YAAY;AAAA,MAC5C,WAAW,YAAY,aAAa,MAAM,cAAc,OAAO;AAC7D,wBAAgB,eAAe,YAAY;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,QAAsC;AACzD,SAAK,QAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,6CAA6C;AAE9F,SAAK,mBAAmB,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,cAAc;AAAA,EAAC;AAAA,EAErB,MAAM,aAAa;AAAA,EAAC;AAAA,EAEpB,CAAS,cAAc,OAA0C;AAC/D,QAAI,KAAK,gBAAgB;AACvB,UAAI,MAAM,eAAe,KAAK,yBAAyB;AAErD,aAAK,iBAAiB;AACtB,aAAK,0BAA0B;AAAA,MACjC;AAAA,IACF;AAEA,QACE,KAAK,mBAAmB,WACvB,MAAM,eAAe,2BAA2B,MAAM,aAAa,uBACpE;AACA,WAAK,iBAAiB,IAAI;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF;AACA,WAAK,0BAA0B,MAAM;AAAA,IACvC;AAEA,QAAI,KAAK,gBAAgB;AAEvB,iBAAW,kBAAkB,KAAK,eAAe,KAAK,KAAK,GAAG;AAC5D,cAAM;AAAA,MACR;AAAA,IACF,OAAO;AACL,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEQ,gBAAgB,UAAmC;AACzD,QAAI,KAAK,wBAAwB;AAC/B,aAAO;AAAA,IACT;AACA,QAAI,SAAS,UAAU;AACrB,aAAO;AAAA,IACT;AAEA,UAAM,gBAAgB,SAAS;AAC/B,WACE,CAAC,CAAC,kBACD,cAAc,aACb,cAAc,uBAAuB,QACrC,cAAc,sBAAsB,QACpC,cAAc,sBAAsB,QACpC,cAAc,gBAAgB;AAAA,EAEpC;AACF;","names":["_a"]}
@@ -227,7 +227,7 @@ export declare class RealtimeSession extends llm.RealtimeSession {
227
227
  private hasReceivedAudioInput;
228
228
  private pendingInterruptText;
229
229
  private earlyCompletionPending;
230
- private toolCallPending;
230
+ private pendingToolCallIds;
231
231
  private generationPendingTurnComplete?;
232
232
  constructor(realtimeModel: RealtimeModel);
233
233
  private closeActiveSession;
@@ -227,7 +227,7 @@ export declare class RealtimeSession extends llm.RealtimeSession {
227
227
  private hasReceivedAudioInput;
228
228
  private pendingInterruptText;
229
229
  private earlyCompletionPending;
230
- private toolCallPending;
230
+ private pendingToolCallIds;
231
231
  private generationPendingTurnComplete?;
232
232
  constructor(realtimeModel: RealtimeModel);
233
233
  private closeActiveSession;
@@ -1 +1 @@
1
- {"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAWL,GAAG,EAKJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAc3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;IAE1B,IAAI,KAAK,IAAI,MAAM,CAElB;gBAGC,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;;;;WAKG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IAgER;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,6BAA6B,CAAC,CAAqB;gBAO/C,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAuBhC,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA8EtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAwBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAkBzB,OAAO,CAAC,mBAAmB;IAIrB,SAAS;IAkBT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAoKd,QAAQ;YA6ER,gBAAgB;IA0G9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IAqDjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAiE1B,OAAO,CAAC,kBAAkB;IAqE1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IA6G3B,OAAO,CAAC,cAAc;IAgCtB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;IA+BtB,OAAO,CAAC,eAAe;CAkBxB"}
1
+ {"version":3,"file":"realtime_api.d.ts","sourceRoot":"","sources":["../../../src/beta/realtime/realtime_api.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,eAAe,CAAC;AACvC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EAEnC,KAAK,WAAW,EAChB,QAAQ,EACR,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAWL,GAAG,EAKJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,UAAU,EAAkB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAc3D;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;CAQxC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,UAAU,eAAe;IACvB,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,QAAQ,EAAE,CAAC;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uBAAuB,CAAC,EAAE,wBAAwB,CAAC;IACnD,wBAAwB,CAAC,EAAE,wBAAwB,CAAC;IACpD,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;IACzD,WAAW,EAAE,iBAAiB,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,QAAQ,CAAC;IACvB,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;CACvC;AA2BD;;GAEG;AACH,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,gBAAgB;IAChB,QAAQ,EAAE,eAAe,CAAC;IAE1B,IAAI,KAAK,IAAI,MAAM,CAElB;gBAGC,OAAO,GAAE;QACP;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;QAE/B;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAEvB;;;WAGG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;QAExB;;WAEG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;QAEnB;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,uBAAuB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE1D;;WAEG;QACH,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QAE3D;;WAEG;QACH,kBAAkB,CAAC,EAAE,OAAO,4BAA4B,CAAC;QAEzD;;WAEG;QACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,OAAO,CAAC;QAEtB;;WAEG;QACH,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;QAE1C;;WAEG;QACH,wBAAwB,CAAC,EAAE,8BAA8B,CAAC;QAE1D;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAEhC;;WAEG;QACH,WAAW,CAAC,EAAE,WAAW,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,QAAQ,CAAC;QAEvB;;;;;WAKG;QACH,cAAc,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC;KAClC;IAgER;;OAEG;IACH,OAAO;IAIP;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAW9E;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,GAAG,CAAC,eAAe;;IACtD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAA2B;IAE3C,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,cAAc,CAAuC;IAC7D,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAC,CAAqB;IAC/C,OAAO,CAAC,OAAO,CAAkB;IAGjC,OAAO,CAAC,aAAa,CAAC,CAAU;IAChC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,sBAAsB,CAA4D;IAC1F,OAAO,CAAC,oBAAoB,CAAC,CAAqC;IAElE,OAAO,CAAC,uBAAuB,CAAC,CAAS;IACzC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,6BAA6B,CAAC,CAAqB;gBAO/C,aAAa,EAAE,aAAa;YAwC1B,kBAAkB;IAuBhC,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,yBAAyB;IAyBjC,aAAa,CAAC,OAAO,EAAE;QACrB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;KAC7B;IAkBK,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvD,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA8EtD,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxD,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,CAE7B;IAED,IAAI,KAAK,IAAI,GAAG,CAAC,WAAW,CAE3B;IAED,IAAI,uBAAuB,IAAI,OAAO,CAErC;IAED,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAwBlC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI;IAI9B,OAAO,CAAC,eAAe;IAIjB,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAyD/E,iBAAiB,IAAI,IAAI;IAkBzB,OAAO,CAAC,mBAAmB;IAIrB,SAAS;IAkBT,QAAQ,CAAC,QAAQ,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE;IAItF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAoKd,QAAQ;YA+ER,gBAAgB;IAyG9B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,mBAAmB;IAmB3B,OAAO,CAAC,qBAAqB;IA4B7B,OAAO,CAAC,yBAAyB;IAqDjC,OAAO,CAAC,SAAS;IAUjB,OAAO,CAAC,kBAAkB;IAiE1B,OAAO,CAAC,kBAAkB;IAqE1B,OAAO,CAAC,wBAAwB;IAIhC,OAAO,CAAC,wBAAwB;IAMhC,OAAO,CAAC,mBAAmB;IA6G3B,OAAO,CAAC,cAAc;IA8BtB,OAAO,CAAC,0BAA0B;IAYlC,OAAO,CAAC,mBAAmB;IA6C3B,OAAO,CAAC,eAAe;IA0BvB,OAAO,CAAC,YAAY;IAMd,WAAW;IAEX,UAAU;IAEhB,OAAO,CAAE,aAAa;IA+BtB,OAAO,CAAC,eAAe;CAkBxB"}
@@ -147,7 +147,7 @@ class RealtimeSession extends llm.RealtimeSession {
147
147
  hasReceivedAudioInput = false;
148
148
  pendingInterruptText = false;
149
149
  earlyCompletionPending = false;
150
- toolCallPending = false;
150
+ pendingToolCallIds = /* @__PURE__ */ new Set();
151
151
  generationPendingTurnComplete;
152
152
  #client;
153
153
  #task;
@@ -193,7 +193,7 @@ class RealtimeSession extends llm.RealtimeSession {
193
193
  }
194
194
  this.earlyCompletionPending = false;
195
195
  this.pendingInterruptText = false;
196
- this.toolCallPending = false;
196
+ this.pendingToolCallIds.clear();
197
197
  if (this.generationPendingTurnComplete) {
198
198
  this.markCurrentGenerationDone(false, this.generationPendingTurnComplete);
199
199
  this.generationPendingTurnComplete = void 0;
@@ -322,7 +322,7 @@ class RealtimeSession extends llm.RealtimeSession {
322
322
  return ((_b = (_a = this.options.realtimeInputConfig) == null ? void 0 : _a.automaticActivityDetection) == null ? void 0 : _b.disabled) ?? false;
323
323
  }
324
324
  pushAudio(frame) {
325
- if (this.toolCallPending) return;
325
+ if (this.pendingToolCallIds.size > 0) return;
326
326
  this.hasReceivedAudioInput = true;
327
327
  for (const f of this.resampleAudio(frame)) {
328
328
  for (const nf of this.bstream.write(f.data.buffer)) {
@@ -397,7 +397,7 @@ class RealtimeSession extends llm.RealtimeSession {
397
397
  if (!this.manualActivityDetection) {
398
398
  return;
399
399
  }
400
- if (this.toolCallPending) return;
400
+ if (this.pendingToolCallIds.size > 0) return;
401
401
  if (!this.inUserActivity) {
402
402
  this.inUserActivity = true;
403
403
  this.sendClientEvent({
@@ -590,13 +590,15 @@ class RealtimeSession extends llm.RealtimeSession {
590
590
  functionResponses
591
591
  });
592
592
  } finally {
593
- this.toolCallPending = false;
593
+ for (const fr of functionResponses) {
594
+ if (fr == null ? void 0 : fr.id) this.pendingToolCallIds.delete(fr.id);
595
+ }
594
596
  }
595
597
  }
596
598
  break;
597
599
  case "realtime_input":
598
600
  const { mediaChunks, activityStart, activityEnd, text } = msg.value;
599
- if (this.toolCallPending) break;
601
+ if (this.pendingToolCallIds.size > 0) break;
600
602
  if (mediaChunks) {
601
603
  for (const mediaChunk of mediaChunks) {
602
604
  await session.sendRealtimeInput({ media: mediaChunk });
@@ -631,7 +633,6 @@ class RealtimeSession extends llm.RealtimeSession {
631
633
  }
632
634
  async onReceiveMessage(session, response) {
633
635
  var _a, _b, _c, _d, _e;
634
- if (response.toolCall) this.toolCallPending = true;
635
636
  const hasAudioData = (_c = (_b = (_a = response.serverContent) == null ? void 0 : _a.modelTurn) == null ? void 0 : _b.parts) == null ? void 0 : _c.some(
636
637
  (part) => {
637
638
  var _a2;
@@ -996,7 +997,6 @@ class RealtimeSession extends llm.RealtimeSession {
996
997
  return;
997
998
  }
998
999
  const gen = this.currentGeneration;
999
- this.toolCallPending = true;
1000
1000
  if (gen.functionChannel.closed) {
1001
1001
  this.#logger.warn("received tool call but functionChannel is already closed.");
1002
1002
  return;
@@ -1006,15 +1006,16 @@ class RealtimeSession extends llm.RealtimeSession {
1006
1006
  this.#logger.warn("received function call without name, skipping");
1007
1007
  continue;
1008
1008
  }
1009
+ const callId = fc.id || shortuuid("fnc-call-");
1010
+ this.pendingToolCallIds.add(callId);
1009
1011
  gen.functionChannel.write(
1010
1012
  llm.FunctionCall.create({
1011
- callId: fc.id || shortuuid("fnc-call-"),
1013
+ callId,
1012
1014
  name: fc.name,
1013
1015
  args: fc.args ? JSON.stringify(fc.args) : ""
1014
1016
  })
1015
1017
  );
1016
1018
  }
1017
- gen.functionChannel.close();
1018
1019
  }
1019
1020
  handleToolCallCancellation(cancellation) {
1020
1021
  this.#logger.warn(
@@ -1023,6 +1024,9 @@ class RealtimeSession extends llm.RealtimeSession {
1023
1024
  },
1024
1025
  "server cancelled tool calls"
1025
1026
  );
1027
+ for (const id of cancellation.ids || []) {
1028
+ this.pendingToolCallIds.delete(id);
1029
+ }
1026
1030
  }
1027
1031
  handleUsageMetadata(usage) {
1028
1032
  if (!this.currentGeneration) {