@livekit/agents-plugin-openai 0.9.2 → 1.0.0-next.0

package/dist/realtime/realtime_model.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AsyncIterableQueue,\n  Future,\n  Queue,\n  llm,\n  log,\n  mergeFrames,\n  metrics,\n  multimodal,\n} from '@livekit/agents';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { once } from 'node:events';\nimport { WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\ninterface ModelOptions {\n  modalities: ['text', 'audio'] | ['text'];\n  instructions: string;\n  voice: api_proto.Voice;\n  inputAudioFormat: api_proto.AudioFormat;\n  outputAudioFormat: api_proto.AudioFormat;\n  inputAudioTranscription: api_proto.InputAudioTranscription | null;\n  turnDetection: api_proto.TurnDetectionType | null;\n  temperature: number;\n  maxResponseOutputTokens: number;\n  model: api_proto.Model;\n  apiKey?: string;\n  baseURL: string;\n  isAzure: boolean;\n  entraToken?: string;\n  apiVersion?: string;\n}\n\nexport interface RealtimeResponse {\n  id: string;\n  status: api_proto.ResponseStatus;\n  statusDetails: api_proto.ResponseStatusDetails | null;\n  usage: api_proto.ModelUsage | null;\n  output: RealtimeOutput[];\n  doneFut: Future;\n  createdTimestamp: number;\n  firstTokenTimestamp?: number;\n}\n\nexport interface RealtimeOutput {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  role: api_proto.Role;\n  type: 'message' | 'function_call';\n  content: RealtimeContent[];\n  doneFut: Future;\n}\n\nexport interface RealtimeContent {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  contentIndex: number;\n  text: string;\n  audio: AudioFrame[];\n  textStream: AsyncIterableQueue<string>;\n  audioStream: AsyncIterableQueue<AudioFrame>;\n  toolCalls: RealtimeToolCall[];\n  contentType: api_proto.Modality;\n}\n\nexport interface RealtimeToolCall {\n  name: string;\n  arguments: string;\n  toolCallID: string;\n}\n\nexport interface InputSpeechTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n}\n\nexport interface InputSpeechTranscriptionFailed {\n  itemId: string;\n  message: string;\n}\n\nexport interface InputSpeechStarted {\n  itemId: string;\n}\n\nexport interface InputSpeechCommitted {\n  itemId: string;\n}\n\nclass InputAudioBuffer {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  append(frame: AudioFrame) {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.append',\n      audio: Buffer.from(frame.data.buffer).toString('base64'),\n    });\n  }\n\n  clear() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.clear',\n    });\n  }\n\n  commit() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.commit',\n    });\n  }\n}\n\nclass ConversationItem {\n  #session: RealtimeSession;\n  #logger = log();\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  truncate(itemId: string, contentIndex: number, audioEnd: number) {\n    this.#session.queueMsg({\n      type: 'conversation.item.truncate',\n      item_id: itemId,\n      content_index: contentIndex,\n      audio_end_ms: audioEnd,\n    });\n  }\n\n  delete(itemId: string) {\n    this.#session.queueMsg({\n      type: 'conversation.item.delete',\n      item_id: itemId,\n    });\n  }\n\n  create(message: llm.ChatMessage, previousItemId?: string): void {\n    if (!message.content) {\n      return;\n    }\n\n    let event: api_proto.ConversationItemCreateEvent;\n\n    if (message.toolCallId) {\n      if (typeof message.content !== 'string') {\n        throw new TypeError('message.content must be a string');\n      }\n\n      event = {\n        type: 'conversation.item.create',\n        previous_item_id: previousItemId,\n        item: {\n          type: 'function_call_output',\n          call_id: message.toolCallId,\n          output: message.content,\n        },\n      };\n    } else {\n      let content = message.content;\n      if (!Array.isArray(content)) {\n        content = [content];\n      }\n\n      if (message.role === llm.ChatRole.USER) {\n        const contents: (api_proto.InputTextContent | api_proto.InputAudioContent)[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            contents.push({\n              type: 'input_audio',\n              audio: Buffer.from(mergeFrames(c.frame).data.buffer).toString('base64'),\n            });\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'user',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.ASSISTANT) {\n        const contents: api_proto.TextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in assistant message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'assistant',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.SYSTEM) {\n        const contents: api_proto.InputTextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in system message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'system',\n            content: contents,\n          },\n        };\n      } else {\n        this.#logger\n          .child({ message })\n          .warn('chat message is not supported inside the realtime API');\n        return;\n      }\n    }\n\n    this.#session.queueMsg(event);\n  }\n}\n\nclass Conversation {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  get item(): ConversationItem {\n    return new ConversationItem(this.#session);\n  }\n}\n\nclass Response {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  create() {\n    this.#session.queueMsg({\n      type: 'response.create',\n    });\n  }\n\n  cancel() {\n    this.#session.queueMsg({\n      type: 'response.cancel',\n    });\n  }\n}\n\ninterface ContentPtr {\n  response_id: string;\n  output_index: number;\n  content_index: number;\n}\n\nexport class RealtimeModel extends multimodal.RealtimeModel {\n  sampleRate = api_proto.SAMPLE_RATE;\n  numChannels = api_proto.NUM_CHANNELS;\n  inFrameSize = api_proto.IN_FRAME_SIZE;\n  outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n  #defaultOpts: ModelOptions;\n  #sessions: RealtimeSession[] = [];\n\n  static withAzure({\n    baseURL,\n    azureDeployment,\n    apiVersion = '2024-10-01-preview',\n    apiKey = undefined,\n    entraToken = undefined,\n    instructions = '',\n    modalities = ['text', 'audio'],\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n  }: {\n    baseURL: string;\n    azureDeployment: string;\n    apiVersion?: string;\n    apiKey?: string;\n    entraToken?: string;\n    instructions?: string;\n    modalities?: ['text', 'audio'] | ['text'];\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }) {\n    return new RealtimeModel({\n      isAzure: true,\n      baseURL: new URL('openai', baseURL).toString(),\n      model: azureDeployment,\n      apiVersion,\n      apiKey,\n      entraToken,\n      instructions,\n      modalities,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n    });\n  }\n\n  constructor({\n    modalities = ['text', 'audio'],\n    instructions = '',\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n    model = 'gpt-4o-realtime-preview-2024-10-01',\n    apiKey = process.env.OPENAI_API_KEY || '',\n    baseURL = api_proto.BASE_URL,\n    // used for microsoft\n    isAzure = false,\n    apiVersion = undefined,\n    entraToken = undefined,\n  }: {\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    model?: api_proto.Model;\n    apiKey?: string;\n    baseURL?: string;\n    isAzure?: boolean;\n    apiVersion?: string;\n    entraToken?: string;\n  }) {\n    super();\n\n    if (apiKey === '' && !(isAzure && entraToken)) {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable',\n      );\n    }\n\n    this.#defaultOpts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model,\n      apiKey,\n      baseURL,\n      isAzure,\n      apiVersion,\n      entraToken,\n    };\n  }\n\n  get sessions(): RealtimeSession[] {\n    return this.#sessions;\n  }\n\n  session({\n    fncCtx,\n    chatCtx,\n    modalities = this.#defaultOpts.modalities,\n    instructions = this.#defaultOpts.instructions,\n    voice = this.#defaultOpts.voice,\n    inputAudioFormat = this.#defaultOpts.inputAudioFormat,\n    outputAudioFormat = this.#defaultOpts.outputAudioFormat,\n    inputAudioTranscription = this.#defaultOpts.inputAudioTranscription,\n    turnDetection = this.#defaultOpts.turnDetection,\n    temperature = this.#defaultOpts.temperature,\n    maxResponseOutputTokens = this.#defaultOpts.maxResponseOutputTokens,\n  }: {\n    fncCtx?: llm.FunctionContext;\n    chatCtx?: llm.ChatContext;\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }): RealtimeSession {\n    const opts: ModelOptions = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#defaultOpts.model,\n      apiKey: this.#defaultOpts.apiKey,\n      baseURL: this.#defaultOpts.baseURL,\n      isAzure: this.#defaultOpts.isAzure,\n      apiVersion: this.#defaultOpts.apiVersion,\n      entraToken: this.#defaultOpts.entraToken,\n    };\n\n    const newSession = new RealtimeSession(opts, {\n      chatCtx: chatCtx || new llm.ChatContext(),\n      fncCtx,\n    });\n    this.#sessions.push(newSession);\n    return newSession;\n  }\n\n  async close() {\n    await Promise.allSettled(this.#sessions.map((session) => session.close()));\n  }\n}\n\nexport class RealtimeSession extends multimodal.RealtimeSession {\n  #chatCtx: llm.ChatContext | undefined = undefined;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #opts: ModelOptions;\n  #pendingResponses: { [id: string]: RealtimeResponse } = {};\n  #sessionId = 'not-connected';\n  #ws: WebSocket | null = null;\n  #expiresAt: number | null = null;\n  #logger = log();\n  #task: Promise<void>;\n  #closing = true;\n  #sendQueue = new Queue<api_proto.ClientEvent>();\n\n  constructor(\n    opts: ModelOptions,\n    { fncCtx, chatCtx }: { fncCtx?: llm.FunctionContext; chatCtx?: llm.ChatContext },\n  ) {\n    super();\n\n    this.#opts = opts;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n\n    this.#task = this.#start();\n\n    this.sessionUpdate({\n      modalities: this.#opts.modalities,\n      instructions: this.#opts.instructions,\n      voice: this.#opts.voice,\n      inputAudioFormat: this.#opts.inputAudioFormat,\n      outputAudioFormat: this.#opts.outputAudioFormat,\n      inputAudioTranscription: this.#opts.inputAudioTranscription,\n      turnDetection: this.#opts.turnDetection,\n      temperature: this.#opts.temperature,\n      maxResponseOutputTokens: this.#opts.maxResponseOutputTokens,\n      toolChoice: 'auto',\n    });\n  }\n\n  get chatCtx(): llm.ChatContext | undefined {\n    return this.#chatCtx;\n  }\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n  }\n\n  get conversation(): Conversation {\n    return new Conversation(this);\n  }\n\n  get inputAudioBuffer(): InputAudioBuffer {\n    return new InputAudioBuffer(this);\n  }\n\n  get response(): Response {\n    return new Response(this);\n  }\n\n  get expiration(): number {\n    if (!this.#expiresAt) {\n      throw new Error('session not started');\n    }\n    return this.#expiresAt * 1000;\n  }\n\n  queueMsg(command: api_proto.ClientEvent): void {\n    this.#sendQueue.put(command);\n  }\n\n  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs\n  /// with large amounts of base64 audio data.\n  #loggableEvent(\n    event: api_proto.ClientEvent | api_proto.ServerEvent,\n    maxLength: number = 30,\n  ): Record<string, unknown> {\n    const untypedEvent: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(event)) {\n      if (value !== undefined) {\n        untypedEvent[key] = value;\n      }\n    }\n\n    if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n      const truncatedData =\n        untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? '…' : '');\n      return { ...untypedEvent, audio: truncatedData };\n    }\n    if (\n      untypedEvent.delta &&\n      typeof untypedEvent.delta === 'string' &&\n      event.type === 'response.audio.delta'\n    ) {\n      const truncatedDelta =\n        untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? '…' : '');\n      return { ...untypedEvent, delta: truncatedDelta };\n    }\n    return untypedEvent;\n  }\n\n  sessionUpdate({\n    modalities = this.#opts.modalities,\n    instructions = this.#opts.instructions,\n    voice = this.#opts.voice,\n    inputAudioFormat = this.#opts.inputAudioFormat,\n    outputAudioFormat = this.#opts.outputAudioFormat,\n    inputAudioTranscription = this.#opts.inputAudioTranscription,\n    turnDetection = this.#opts.turnDetection,\n    temperature = this.#opts.temperature,\n    maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,\n    toolChoice = 'auto',\n    selectedTools = Object.keys(this.#fncCtx || {}),\n  }: {\n    modalities: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    toolChoice?: api_proto.ToolChoice;\n    selectedTools?: string[];\n  }) {\n    this.#opts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#opts.model,\n      apiKey: this.#opts.apiKey,\n      baseURL: this.#opts.baseURL,\n      isAzure: this.#opts.isAzure,\n      apiVersion: this.#opts.apiVersion,\n      entraToken: this.#opts.entraToken,\n    };\n\n    const tools = this.#fncCtx\n      ? Object.entries(this.#fncCtx)\n          .filter(([name]) => selectedTools.includes(name))\n          .map(([name, func]) => ({\n            type: 'function' as const,\n            name,\n            description: func.description,\n            parameters:\n              // don't format parameters if they are raw openai params\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          }))\n      : [];\n\n    const sessionUpdateEvent: api_proto.SessionUpdateEvent = {\n      type: 'session.update',\n      session: {\n        modalities: this.#opts.modalities,\n        instructions: this.#opts.instructions,\n        voice: this.#opts.voice,\n        input_audio_format: this.#opts.inputAudioFormat,\n        output_audio_format: this.#opts.outputAudioFormat,\n        input_audio_transcription: this.#opts.inputAudioTranscription,\n        turn_detection: this.#opts.turnDetection,\n        temperature: this.#opts.temperature,\n        max_response_output_tokens:\n          this.#opts.maxResponseOutputTokens === Infinity\n            ? 'inf'\n            : this.#opts.maxResponseOutputTokens,\n        tools,\n        tool_choice: toolChoice,\n      },\n    };\n\n    if (this.#opts.isAzure && this.#opts.maxResponseOutputTokens === Infinity) {\n      // microsoft doesn't support inf for max_response_output_tokens, but accepts no args\n      sessionUpdateEvent.session.max_response_output_tokens = undefined;\n    }\n\n    this.queueMsg(sessionUpdateEvent);\n  }\n\n  /** Create an empty audio message with the given duration. */\n  #createEmptyUserAudioMessage(duration: number): llm.ChatMessage {\n    const samples = duration * api_proto.SAMPLE_RATE;\n    return new llm.ChatMessage({\n      role: llm.ChatRole.USER,\n      content: {\n        frame: new AudioFrame(\n          new Int16Array(samples * api_proto.NUM_CHANNELS),\n          api_proto.SAMPLE_RATE,\n          api_proto.NUM_CHANNELS,\n          samples,\n        ),\n      },\n    });\n  }\n\n  /**\n   * Try to recover from a text response to audio mode.\n   *\n   * @remarks\n   * Sometimes the OpenAI Realtime API returns text instead of audio responses.\n   * This method tries to recover from this by requesting a new response after deleting the text\n   * response and creating an empty user audio message.\n   */\n  recoverFromTextResponse(itemId: string) {\n    if (itemId) {\n      this.conversation.item.delete(itemId);\n    }\n    this.conversation.item.create(this.#createEmptyUserAudioMessage(1));\n    this.response.create();\n  }\n\n  #start(): Promise<void> {\n    return new Promise(async (resolve, reject) => {\n      const headers: Record<string, string> = {\n        'User-Agent': 'LiveKit-Agents-JS',\n      };\n      if (this.#opts.isAzure) {\n        // Microsoft API has two ways of authentication\n        // 1. Entra token set as `Bearer` token\n        // 2. API key set as `api_key` header (also accepts query string)\n        if (this.#opts.entraToken) {\n          headers.Authorization = `Bearer ${this.#opts.entraToken}`;\n        } else if (this.#opts.apiKey) {\n          headers['api-key'] = this.#opts.apiKey;\n        } else {\n          reject(new Error('Microsoft API key or entraToken is required'));\n          return;\n        }\n      } else {\n        headers.Authorization = `Bearer ${this.#opts.apiKey}`;\n        headers['OpenAI-Beta'] = 'realtime=v1';\n      }\n      const url = new URL([this.#opts.baseURL, 'realtime'].join('/'));\n      if (url.protocol === 'https:') {\n        url.protocol = 'wss:';\n      }\n\n      // Construct query parameters\n      const queryParams: Record<string, string> = {};\n      if (this.#opts.isAzure) {\n        queryParams['api-version'] = this.#opts.apiVersion ?? '2024-10-01-preview';\n        queryParams['deployment'] = this.#opts.model;\n      } else {\n        queryParams['model'] = this.#opts.model;\n      }\n\n      for (const [key, value] of Object.entries(queryParams)) {\n        url.searchParams.set(key, value);\n      }\n\n      console.debug('Connecting to OpenAI Realtime API at ', url.toString());\n      this.#ws = new WebSocket(url.toString(), {\n        headers: headers,\n      });\n\n      this.#ws.onerror = (error) => {\n        reject(new Error('OpenAI Realtime WebSocket error: ' + error.message));\n      };\n\n      await once(this.#ws, 'open');\n      this.#closing = false;\n\n      this.#ws.onmessage = (message) => {\n        const event: api_proto.ServerEvent = JSON.parse(message.data as string);\n        this.#logger.debug(`<- ${JSON.stringify(this.#loggableEvent(event))}`);\n        switch (event.type) {\n          case 'error':\n            this.#handleError(event);\n            break;\n          case 'session.created':\n            this.#handleSessionCreated(event);\n            break;\n          case 'session.updated':\n            this.#handleSessionUpdated(event);\n            break;\n          case 'conversation.created':\n            this.#handleConversationCreated(event);\n            break;\n          case 'input_audio_buffer.committed':\n            this.#handleInputAudioBufferCommitted(event);\n            break;\n          case 'input_audio_buffer.cleared':\n            this.#handleInputAudioBufferCleared(event);\n            break;\n          case 'input_audio_buffer.speech_started':\n            this.#handleInputAudioBufferSpeechStarted(event);\n            break;\n          case 'input_audio_buffer.speech_stopped':\n            this.#handleInputAudioBufferSpeechStopped(event);\n            break;\n          case 'conversation.item.created':\n            this.#handleConversationItemCreated(event);\n            break;\n          case 'conversation.item.input_audio_transcription.completed':\n            this.#handleConversationItemInputAudioTranscriptionCompleted(event);\n            break;\n          case 'conversation.item.input_audio_transcription.failed':\n            this.#handleConversationItemInputAudioTranscriptionFailed(event);\n            break;\n          case 'conversation.item.truncated':\n            this.#handleConversationItemTruncated(event);\n            break;\n          case 'conversation.item.deleted':\n            this.#handleConversationItemDeleted(event);\n            break;\n          case 'response.created':\n            this.#handleResponseCreated(event);\n            break;\n          case 'response.done':\n            this.#handleResponseDone(event);\n            break;\n          case 'response.output_item.added':\n            this.#handleResponseOutputItemAdded(event);\n            break;\n          case 'response.output_item.done':\n            this.#handleResponseOutputItemDone(event);\n            break;\n          case 'response.content_part.added':\n            this.#handleResponseContentPartAdded(event);\n            break;\n          case 'response.content_part.done':\n            this.#handleResponseContentPartDone(event);\n            break;\n          case 'response.text.delta':\n            this.#handleResponseTextDelta(event);\n            break;\n          case 'response.text.done':\n            this.#handleResponseTextDone(event);\n            break;\n          case 'response.audio_transcript.delta':\n            this.#handleResponseAudioTranscriptDelta(event);\n            break;\n          case 'response.audio_transcript.done':\n            this.#handleResponseAudioTranscriptDone(event);\n            break;\n          case 'response.audio.delta':\n            this.#handleResponseAudioDelta(event);\n            break;\n          case 'response.audio.done':\n            this.#handleResponseAudioDone(event);\n            break;\n          case 'response.function_call_arguments.delta':\n            this.#handleResponseFunctionCallArgumentsDelta(event);\n            break;\n          case 'response.function_call_arguments.done':\n            this.#handleResponseFunctionCallArgumentsDone(event);\n            break;\n          case 'rate_limits.updated':\n            this.#handleRateLimitsUpdated(event);\n            break;\n        }\n      };\n\n      const sendTask = async () => {\n        while (this.#ws && !this.#closing && this.#ws.readyState === WebSocket.OPEN) {\n          try {\n            const event = await this.#sendQueue.get();\n            if (event.type !== 'input_audio_buffer.append') {\n              this.#logger.debug(`-> ${JSON.stringify(this.#loggableEvent(event))}`);\n            }\n            this.#ws.send(JSON.stringify(event));\n          } catch (error) {\n            this.#logger.error('Error sending event:', error);\n          }\n        }\n      };\n\n      sendTask();\n\n      this.#ws.onclose = () => {\n        if (this.#expiresAt && Date.now() >= this.#expiresAt * 1000) {\n          this.#closing = true;\n        }\n        if (!this.#closing) {\n          reject(new Error('OpenAI Realtime connection closed unexpectedly'));\n        }\n        this.#ws = null;\n        resolve();\n      };\n    });\n  }\n\n  async close() {\n    if (!this.#ws) return;\n    this.#closing = true;\n    this.#ws.close();\n    await this.#task;\n  }\n\n  #getContent(ptr: ContentPtr): RealtimeContent {\n    const response = this.#pendingResponses[ptr.response_id];\n    const output = response!.output[ptr.output_index];\n    const content = output!.content[ptr.content_index]!;\n    return content;\n  }\n\n  #handleError(event: api_proto.ErrorEvent): void {\n    this.#logger.error(`OpenAI Realtime error ${JSON.stringify(event.error)}`);\n  }\n\n  #handleSessionCreated(event: api_proto.SessionCreatedEvent): void {\n    this.#sessionId = event.session.id;\n    this.#expiresAt = event.session.expires_at;\n    this.#logger = this.#logger.child({ sessionId: this.#sessionId });\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleSessionUpdated(event: api_proto.SessionUpdatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationCreated(event: api_proto.ConversationCreatedEvent): void {}\n\n  #handleInputAudioBufferCommitted(event: api_proto.InputAudioBufferCommittedEvent): void {\n    this.emit('input_speech_committed', {\n      itemId: event.item_id,\n    } as InputSpeechCommitted);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleInputAudioBufferCleared(event: api_proto.InputAudioBufferClearedEvent): void {}\n\n  #handleInputAudioBufferSpeechStarted(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStartedEvent,\n  ): void {\n    this.emit('input_speech_started', {\n      itemId: event.item_id,\n    } as InputSpeechStarted);\n  }\n\n  #handleInputAudioBufferSpeechStopped(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStoppedEvent,\n  ): void {\n    this.emit('input_speech_stopped');\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {}\n\n  #handleConversationItemInputAudioTranscriptionCompleted(\n    event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n  ): void {\n    const transcript = event.transcript;\n    this.emit('input_speech_transcription_completed', {\n      itemId: event.item_id,\n      transcript: transcript,\n    } as InputSpeechTranscriptionCompleted);\n  }\n\n  #handleConversationItemInputAudioTranscriptionFailed(\n    event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n  ): void {\n    const error = event.error;\n    this.#logger.error(`OpenAI Realtime failed to transcribe input audio: ${error.message}`);\n    this.emit('input_speech_transcription_failed', {\n      itemId: event.item_id,\n      message: error.message,\n    } as InputSpeechTranscriptionFailed);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemTruncated(event: api_proto.ConversationItemTruncatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {}\n\n  #handleResponseCreated(responseCreated: api_proto.ResponseCreatedEvent): void {\n    const response = responseCreated.response;\n    const doneFut = new Future();\n    const newResponse: RealtimeResponse = {\n      id: response.id,\n      status: response.status,\n      statusDetails: response.status_details,\n      usage: null,\n      output: [],\n      doneFut: doneFut,\n      createdTimestamp: Date.now(),\n    };\n    this.#pendingResponses[newResponse.id] = newResponse;\n    this.emit('response_created', newResponse);\n  }\n\n  #handleResponseDone(event: api_proto.ResponseDoneEvent): void {\n    const responseData = event.response;\n    const responseId = responseData.id;\n    const response = this.#pendingResponses[responseId]!;\n    response.status = responseData.status;\n    response.statusDetails = responseData.status_details;\n    response.usage = responseData.usage ?? null;\n    this.#pendingResponses[responseId] = response;\n    response.doneFut.resolve();\n\n    let metricsError: Error | undefined;\n    let cancelled = false;\n    switch (response.status) {\n      case 'failed': {\n        if (response.statusDetails.type !== 'failed') break;\n        const err = response.statusDetails.error;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          code: err?.code,\n          message: err?.message,\n        });\n        this.#logger\n          .child({ code: err?.code, error: err?.message })\n          .error('response generation failed');\n        break;\n      }\n      case 'incomplete': {\n        if (response.statusDetails.type !== 'incomplete') break;\n        const reason = response.statusDetails.reason;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          reason,\n        });\n        this.#logger.child({ reason }).error('response generation incomplete');\n        break;\n      }\n      case 'cancelled': {\n        cancelled = true;\n        break;\n      }\n    }\n    this.emit('response_done', response);\n\n    let ttft: number | undefined;\n    if (response.firstTokenTimestamp) {\n      ttft = response.firstTokenTimestamp - response.createdTimestamp;\n    }\n    const duration = Date.now() - response.createdTimestamp;\n\n    const usage = response.usage;\n    const metric: metrics.MultimodalLLMMetrics = {\n      timestamp: response.createdTimestamp,\n      requestId: response.id,\n      ttft: ttft!,\n      duration,\n      cancelled,\n      label: this.constructor.name,\n      completionTokens: usage?.output_tokens || 0,\n      promptTokens: usage?.input_tokens || 0,\n      totalTokens: usage?.total_tokens || 0,\n      tokensPerSecond: ((usage?.output_tokens || 0) / duration) * 1000,\n      error: metricsError,\n      inputTokenDetails: {\n        cachedTokens: usage?.input_token_details.cached_tokens || 0,\n        textTokens: usage?.input_token_details.text_tokens || 0,\n        audioTokens: usage?.input_token_details.audio_tokens || 0,\n      },\n      outputTokenDetails: {\n        textTokens: usage?.output_token_details.text_tokens || 0,\n        audioTokens: usage?.output_token_details.audio_tokens || 0,\n      },\n    };\n    this.emit('metrics_collected', metric);\n  }\n\n  #handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const itemData = event.item;\n\n    if (itemData.type !== 'message' && itemData.type !== 'function_call') {\n      throw new Error(`Unexpected item type: ${itemData.type}`);\n    }\n\n    let role: api_proto.Role;\n    if (itemData.type === 'function_call') {\n      role = 'assistant'; // function_call doesn't have a role field, defaulting it to assistant\n    } else {\n      role = itemData.role;\n    }\n\n    const newOutput: RealtimeOutput = {\n      responseId: responseId,\n      itemId: itemData.id,\n      outputIndex: event.output_index,\n      type: itemData.type,\n      role: role,\n      content: [],\n      doneFut: new Future(),\n    };\n    response?.output.push(newOutput);\n    this.emit('response_output_added', newOutput);\n  }\n\n  #handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    if (output?.type === 'function_call') {\n      if (!this.#fncCtx) {\n        this.#logger.error('function call received but no fncCtx is available');\n        return;\n      }\n\n      // parse the arguments and call the function inside the fnc_ctx\n      const item = event.item;\n      if (item.type !== 'function_call') {\n        throw new Error('Expected function_call item');\n      }\n      const func = this.#fncCtx[item.name];\n      if (!func) {\n        this.#logger.error(`no function with name ${item.name} in fncCtx`);\n        return;\n      }\n\n      this.emit('function_call_started', {\n        callId: item.call_id,\n      });\n\n      const parsedArgs = JSON.parse(item.arguments);\n\n      this.#logger.debug(\n        `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`,\n      );\n\n      func.execute(parsedArgs).then(\n        (content) => {\n          this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);\n          this.emit('function_call_completed', {\n            callId: item.call_id,\n          });\n          this.conversation.item.create(\n            llm.ChatMessage.createToolFromFunctionResult({\n              name: item.name,\n              toolCallId: item.call_id,\n              result: content,\n            }),\n            output.itemId,\n          );\n          this.response.create();\n        },\n        (error) => {\n          this.#logger.error(`[Function Call ${item.call_id}] ${item.name} failed with ${error}`);\n          // TODO: send it back up as failed?\n          this.emit('function_call_failed', {\n            callId: item.call_id,\n          });\n        },\n      );\n    }\n\n    output?.doneFut.resolve();\n    this.emit('response_output_done', output);\n  }\n\n  #handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    const textStream = new AsyncIterableQueue<string>();\n    const audioStream = new AsyncIterableQueue<AudioFrame>();\n\n    const newContent: RealtimeContent = {\n      responseId: responseId,\n      itemId: event.item_id,\n      outputIndex: outputIndex,\n      contentIndex: event.content_index,\n      text: '',\n      audio: [],\n      textStream: textStream,\n      audioStream: audioStream,\n      toolCalls: [],\n      contentType: event.part.type,\n    };\n    output?.content.push(newContent);\n    response!.firstTokenTimestamp = Date.now();\n    this.emit('response_content_added', newContent);\n  }\n\n  #handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n    const content = this.#getContent(event);\n    this.emit('response_content_done', content);\n  }\n\n  #handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n    this.emit('response_text_delta', event);\n  }\n\n  #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {\n    const content = this.#getContent(event);\n    content.text = event.text;\n    this.emit('response_text_done', event);\n  }\n\n  #handleResponseAudioTranscriptDelta(event: api_proto.ResponseAudioTranscriptDeltaEvent): void {\n    const content = this.#getContent(event);\n    const transcript = event.delta;\n    content.text += transcript;\n\n    content.textStream.put(transcript);\n  }\n\n  #handleResponseAudioTranscriptDone(event: api_proto.ResponseAudioTranscriptDoneEvent): void {\n    const content = this.#getContent(event);\n    content.textStream.close();\n  }\n\n  #handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n    const content = this.#getContent(event);\n    const data = Buffer.from(event.delta, 'base64');\n    const audio = new AudioFrame(\n      new Int16Array(data.buffer),\n      api_proto.SAMPLE_RATE,\n      api_proto.NUM_CHANNELS,\n      data.length / 2,\n    );\n    content.audio.push(audio);\n\n    content.audioStream.put(audio);\n  }\n\n  #handleResponseAudioDone(event: api_proto.ResponseAudioDoneEvent): void {\n    const content = this.#getContent(event);\n    content.audioStream.close();\n  }\n\n  #handleResponseFunctionCallArgumentsDelta(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDeltaEvent,\n  ): void {}\n\n  #handleResponseFunctionCallArgumentsDone(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDoneEvent,\n  ): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleRateLimitsUpdated(event: api_proto.RateLimitsUpdatedEvent): void {}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBASO;AACP,sBAA2B;AAC3B,yBAAqB;AACrB,gBAA0B;AAC1B,gBAA2B;AA8E3B,MAAM,iBAAiB;AAAA,EACrB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,OAAO,OAAmB;AACxB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,OAAO,OAAO,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,IACzD,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAEA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA,cAAU,mBAAI;AAAA,EAEd,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS,QAAgB,cAAsB,UAAkB;AAC/D,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,MACT,eAAe;AAAA,MACf,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,QAAgB;AACrB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,SAA0B,gBAA+B;AAC9D,QAAI,CAAC,QAAQ,SAAS;AACpB;AAAA,IACF;AAEA,QAAI;AAEJ,QAAI,QAAQ,YAAY;AACtB,UAAI,OAAO,QAAQ,YAAY,UAAU;AACvC,cAAM,IAAI,UAAU,kCAAkC;AAAA,MACxD;AAEA,cAAQ;AAAA,QACN,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,SAAS,QAAQ;AAAA,UACjB,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,IACF,OAAO;AACL,UAAI,UAAU,QAAQ;AACtB,UAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B,kBAAU,CAAC,OAAO;AAAA,MACpB;AAEA,UAAI,QAAQ,SAAS,kBAAI,SAAS,MAAM;AACtC,cAAM,WAAyE,CAAC;AAChF,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,OAAO,OAAO,SAAK,2BAAY,EAAE,KAAK,EAAE,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,kBAAI,SAAS,WAAW;AAClD,cAAM,WAAoC,CAAC;AAC3C,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,qDAAqD;AAAA,UACzE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,kBAAI,SAAS,QAAQ;AAC/C,cAAM,WAAyC,CAAC;AAChD,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,kDAAkD;AAAA,UACtE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,OAAO;AACL,aAAK,QACF,MAAM,EAAE,QAAQ,CAAC,EACjB,KAAK,uDAAuD;AAC/D;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,SAAS,KAAK;AAAA,EAC9B;AACF;AAEA,MAAM,aAAa;AAAA,EACjB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,IAAI,OAAyB;AAC3B,WAAO,IAAI,iBAAiB,KAAK,QAAQ;AAAA,EAC3C;AACF;AAEA,MAAM,SAAS;AAAA,EACb;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAQO,MAAM,sBAAsB,yBAAW,cAAc;AAAA,EAC1D,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA,EAEzB;AAAA,EACA,YAA+B,CAAC;AAAA,EAEhC,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA,aAAa;AAAA,IACb,SAAS;AAAA,IACT,aAAa;AAAA,IACb,eAAe;AAAA,IACf,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,EAC5B,GAeG;AACD,WAAO,IAAI,cAAc;AAAA,MACvB,SAAS;AAAA,MACT,SAAS,IAAI,IAAI,UAAU,OAAO,EAAE,SAAS;AAAA,MAC7C,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,YAAY;AAAA,IACV,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,IAC1B,QAAQ;AAAA,IACR,SAAS,QAAQ,IAAI,kBAAkB;AAAA,IACvC,UAAU,UAAU;AAAA;AAAA,IAEpB,UAAU;AAAA,IACV,aAAa;AAAA,IACb,aAAa;AAAA,EACf,GAgBG;AACD,UAAM;AAEN,QAAI,WAAW,MAAM,EAAE,WAAW,aAAa;AAC7C,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,SAAK,eAAe;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,WAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AAAA,IACN;AAAA,IACA;AAAA,IACA,aAAa,KAAK,aAAa;AAAA,IAC/B,eAAe,KAAK,aAAa;AAAA,IACjC,QAAQ,KAAK,aAAa;AAAA,IAC1B,mBAAmB,KAAK,aAAa;AAAA,IACrC,oBAAoB,KAAK,aAAa;AAAA,IACtC,0BAA0B,KAAK,aAAa;AAAA,IAC5C,gBAAgB,KAAK,aAAa;AAAA,IAClC,cAAc,KAAK,aAAa;AAAA,IAChC,0BAA0B,KAAK,aAAa;AAAA,EAC9C,GAYoB;AAClB,UAAM,OAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,aAAa;AAAA,MACzB,QAAQ,KAAK,aAAa;AAAA,MAC1B,SAAS,KAAK,aAAa;AAAA,MAC3B,SAAS,KAAK,aAAa;AAAA,MAC3B,YAAY,KAAK,aAAa;AAAA,MAC9B,YAAY,KAAK,aAAa;AAAA,IAChC;AAEA,UAAM,aAAa,IAAI,gBAAgB,MAAM;AAAA,MAC3C,SAAS,WAAW,IAAI,kBAAI,YAAY;AAAA,MACxC;AAAA,IACF,CAAC;AACD,SAAK,UAAU,KAAK,UAAU;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,QAAQ,WAAW,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC3E;AACF;AAEO,MAAM,wBAAwB,yBAAW,gBAAgB;AAAA,EAC9D,WAAwC;AAAA,EACxC,UAA2C;AAAA,EAC3C;AAAA,EACA,oBAAwD,CAAC;AAAA,EACzD,aAAa;AAAA,EACb,MAAwB;AAAA,EACxB,aAA4B;AAAA,EAC5B,cAAU,mBAAI;AAAA,EACd;AAAA,EACA,WAAW;AAAA,EACX,aAAa,IAAI,oBAA6B;AAAA,EAE9C,YACE,MACA,EAAE,QAAQ,QAAQ,GAClB;AACA,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAEf,SAAK,QAAQ,KAAK,OAAO;AAEzB,SAAK,cAAc;AAAA,MACjB,YAAY,KAAK,MAAM;AAAA,MACvB,cAAc,KAAK,MAAM;AAAA,MACzB,OAAO,KAAK,MAAM;AAAA,MAClB,kBAAkB,KAAK,MAAM;AAAA,MAC7B,mBAAmB,KAAK,MAAM;AAAA,MAC9B,yBAAyB,KAAK,MAAM;AAAA,MACpC,eAAe,KAAK,MAAM;AAAA,MAC1B,aAAa,KAAK,MAAM;AAAA,MACxB,yBAAyB,KAAK,MAAM;AAAA,MACpC,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAAA,EAEA,IAAI,UAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,IAAI,eAA6B;AAC/B,WAAO,IAAI,aAAa,IAAI;AAAA,EAC9B;AAAA,EAEA,IAAI,mBAAqC;AACvC,WAAO,IAAI,iBAAiB,IAAI;AAAA,EAClC;AAAA,EAEA,IAAI,WAAqB;AACvB,WAAO,IAAI,SAAS,IAAI;AAAA,EAC1B;AAAA,EAEA,IAAI,aAAqB;AACvB,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,qBAAqB;AAAA,IACvC;AACA,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,SAAS,SAAsC;AAC7C,SAAK,WAAW,IAAI,OAAO;AAAA,EAC7B;AAAA;AAAA;AAAA,EAIA,eACE,OACA,YAAoB,IACK;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,YAAM,gBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,cAAc;AAAA,IACjD;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,YAAM,iBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,eAAe;AAAA,IAClD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc;AAAA,IACZ,aAAa,KAAK,MAAM;AAAA,IACxB,eAAe,KAAK,MAAM;AAAA,IAC1B,QAAQ,KAAK,MAAM;AAAA,IACnB,mBAAmB,KAAK,MAAM;AAAA,IAC9B,oBAAoB,KAAK,MAAM;AAAA,IAC/B,0BAA0B,KAAK,MAAM;AAAA,IACrC,gBAAgB,KAAK,MAAM;AAAA,IAC3B,cAAc,KAAK,MAAM;AAAA,IACzB,0BAA0B,KAAK,MAAM;AAAA,IACrC,aAAa;AAAA,IACb,gBAAgB,OAAO,KAAK,KAAK,WAAW,CAAC,CAAC;AAAA,EAChD,GAYG;AACD,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,MAAM;AAAA,MAClB,QAAQ,KAAK,MAAM;AAAA,MACnB,SAAS,KAAK,MAAM;AAAA,MACpB,SAAS,KAAK,MAAM;AAAA,MACpB,YAAY,KAAK,MAAM;AAAA,MACvB,YAAY,KAAK,MAAM;AAAA,IACzB;AAEA,UAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EACxB,OAAO,CAAC,CAAC,IAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACtB,MAAM;AAAA,MACN;AAAA,MACA,aAAa,KAAK;AAAA,MAClB;AAAA;AAAA,QAEE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,kBAAI,UAAU,KAAK,UAAU;AAAA;AAAA,IACrC,EAAE,IACJ,CAAC;AAEL,UAAM,qBAAmD;AAAA,MACvD,MAAM;AAAA,MACN,SAAS;AAAA,QACP,YAAY,KAAK,MAAM;AAAA,QACvB,cAAc,KAAK,MAAM;AAAA,QACzB,OAAO,KAAK,MAAM;AAAA,QAClB,oBAAoB,KAAK,MAAM;AAAA,QAC/B,qBAAqB,KAAK,MAAM;AAAA,QAChC,2BAA2B,KAAK,MAAM;AAAA,QACtC,gBAAgB,KAAK,MAAM;AAAA,QAC3B,aAAa,KAAK,MAAM;AAAA,QACxB,4BACE,KAAK,MAAM,4BAA4B,WACnC,QACA,KAAK,MAAM;AAAA,QACjB;AAAA,QACA,aAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,WAAW,KAAK,MAAM,4BAA4B,UAAU;AAEzE,yBAAmB,QAAQ,6BAA6B;AAAA,IAC1D;AAEA,SAAK,SAAS,kBAAkB;AAAA,EAClC;AAAA;AAAA,EAGA,6BAA6B,UAAmC;AAC9D,UAAM,UAAU,WAAW,UAAU;AACrC,WAAO,IAAI,kBAAI,YAAY;AAAA,MACzB,MAAM,kBAAI,SAAS;AAAA,MACnB,SAAS;AAAA,QACP,OAAO,IAAI;AAAA,UACT,IAAI,WAAW,UAAU,UAAU,YAAY;AAAA,UAC/C,UAAU;AAAA,UACV,UAAU;AAAA,UACV;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,wBAAwB,QAAgB;AACtC,QAAI,QAAQ;AACV,WAAK,aAAa,KAAK,OAAO,MAAM;AAAA,IACtC;AACA,SAAK,aAAa,KAAK,OAAO,KAAK,6BAA6B,CAAC,CAAC;AAClE,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,SAAwB;AACtB,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAC5C,YAAM,UAAkC;AAAA,QACtC,cAAc;AAAA,MAChB;AACA,UAAI,KAAK,MAAM,SAAS;AAItB,YAAI,KAAK,MAAM,YAAY;AACzB,kBAAQ,gBAAgB,UAAU,KAAK,MAAM,UAAU;AAAA,QACzD,WAAW,KAAK,MAAM,QAAQ;AAC5B,kBAAQ,SAAS,IAAI,KAAK,MAAM;AAAA,QAClC,OAAO;AACL,iBAAO,IAAI,MAAM,6CAA6C,CAAC;AAC/D;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,gBAAgB,UAAU,KAAK,MAAM,MAAM;AACnD,gBAAQ,aAAa,IAAI;AAAA,MAC3B;AACA,YAAM,MAAM,IAAI,IAAI,CAAC,KAAK,MAAM,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAC9D,UAAI,IAAI,aAAa,UAAU;AAC7B,YAAI,WAAW;AAAA,MACjB;AAGA,YAAM,cAAsC,CAAC;AAC7C,UAAI,KAAK,MAAM,SAAS;AACtB,oBAAY,aAAa,IAAI,KAAK,MAAM,cAAc;AACtD,oBAAY,YAAY,IAAI,KAAK,MAAM;AAAA,MACzC,OAAO;AACL,oBAAY,OAAO,IAAI,KAAK,MAAM;AAAA,MACpC;AAEA,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,YAAI,aAAa,IAAI,KAAK,KAAK;AAAA,MACjC;AAEA,cAAQ,MAAM,yCAAyC,IAAI,SAAS,CAAC;AACrE,WAAK,MAAM,IAAI,oBAAU,IAAI,SAAS,GAAG;AAAA,QACvC;AAAA,MACF,CAAC;AAED,WAAK,IAAI,UAAU,CAAC,UAAU;AAC5B,eAAO,IAAI,MAAM,sCAAsC,MAAM,OAAO,CAAC;AAAA,MACvE;AAEA,gBAAM,yBAAK,KAAK,KAAK,MAAM;AAC3B,WAAK,WAAW;AAEhB,WAAK,IAAI,YAAY,CAAC,YAAY;AAChC,cAAM,QAA+B,KAAK,MAAM,QAAQ,IAAc;AACtE,aAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AACrE,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,aAAa,KAAK;AACvB;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,2BAA2B,KAAK;AACrC;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,wDAAwD,KAAK;AAClE;AAAA,UACF,KAAK;AACH,iBAAK,qDAAqD,KAAK;AAC/D;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,uBAAuB,KAAK;AACjC;AAAA,UACF,KAAK;AACH,iBAAK,oBAAoB,KAAK;AAC9B;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,8BAA8B,KAAK;AACxC;AAAA,UACF,KAAK;AACH,iBAAK,gCAAgC,KAAK;AAC1C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,wBAAwB,KAAK;AAClC;AAAA,UACF,KAAK;AACH,iBAAK,oCAAoC,KAAK;AAC9C;AAAA,UACF,KAAK;AACH,iBAAK,mCAAmC,KAAK;AAC7C;AAAA,UACF,KAAK;AACH,iBAAK,0BAA0B,KAAK;AACpC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,0CAA0C,KAAK;AACpD;AAAA,UACF,KAAK;AACH,iBAAK,yCAAyC,KAAK;AACnD;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,QACJ;AAAA,MACF;AAEA,YAAM,WAAW,YAAY;AAC3B,eAAO,KAAK,OAAO,CAAC,KAAK,YAAY,KAAK,IAAI,eAAe,oBAAU,MAAM;AAC3E,cAAI;AACF,kBAAM,QAAQ,MAAM,KAAK,WAAW,IAAI;AACxC,gBAAI,MAAM,SAAS,6BAA6B;AAC9C,mBAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AAAA,YACvE;AACA,iBAAK,IAAI,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,UACrC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,wBAAwB,KAAK;AAAA,UAClD;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAET,WAAK,IAAI,UAAU,MAAM;AACvB,YAAI,KAAK,cAAc,KAAK,IAAI,KAAK,KAAK,aAAa,KAAM;AAC3D,eAAK,WAAW;AAAA,QAClB;AACA,YAAI,CAAC,KAAK,UAAU;AAClB,iBAAO,IAAI,MAAM,gDAAgD,CAAC;AAAA,QACpE;AACA,aAAK,MAAM;AACX,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,CAAC,KAAK,IAAK;AACf,SAAK,WAAW;AAChB,SAAK,IAAI,MAAM;AACf,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,YAAY,KAAkC;AAC5C,UAAM,WAAW,KAAK,kBAAkB,IAAI,WAAW;AACvD,UAAM,SAAS,SAAU,OAAO,IAAI,YAAY;AAChD,UAAM,UAAU,OAAQ,QAAQ,IAAI,aAAa;AACjD,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmC;AAC9C,SAAK,QAAQ,MAAM,yBAAyB,KAAK,UAAU,MAAM,KAAK,CAAC,EAAE;AAAA,EAC3E;AAAA,EAEA,sBAAsB,OAA4C;AAChE,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,UAAU,KAAK,QAAQ,MAAM,EAAE,WAAW,KAAK,WAAW,CAAC;AAAA,EAClE;AAAA;AAAA,EAGA,sBAAsB,OAA4C;AAAA,EAAC;AAAA;AAAA,EAGnE,2BAA2B,OAAiD;AAAA,EAAC;AAAA,EAE7E,iCAAiC,OAAuD;AACtF,SAAK,KAAK,0BAA0B;AAAA,MAClC,QAAQ,MAAM;AAAA,IAChB,CAAyB;AAAA,EAC3B;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,qCAEE,OACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,QAAQ,MAAM;AAAA,IAChB,CAAuB;AAAA,EACzB;AAAA,EAEA,qCAEE,OACM;AACN,SAAK,KAAK,sBAAsB;AAAA,EAClC;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,wDACE,OACM;AACN,UAAM,aAAa,MAAM;AACzB,SAAK,KAAK,wCAAwC;AAAA,MAChD,QAAQ,MAAM;AAAA,MACd;AAAA,IACF,CAAsC;AAAA,EACxC;AAAA,EAEA,qDACE,OACM;AACN,UAAM,QAAQ,MAAM;AACpB,SAAK,QAAQ,MAAM,qDAAqD,MAAM,OAAO,EAAE;AACvF,SAAK,KAAK,qCAAqC;AAAA,MAC7C,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM;AAAA,IACjB,CAAmC;AAAA,EACrC;AAAA;AAAA,EAGA,iCAAiC,OAAuD;AAAA,EAAC;AAAA;AAAA,EAGzF,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,uBAAuB,iBAAuD;AAC5E,UAAM,WAAW,gBAAgB;AACjC,UAAM,UAAU,IAAI,qBAAO;AAC3B,UAAM,cAAgC;AAAA,MACpC,IAAI,SAAS;AAAA,MACb,QAAQ,SAAS;AAAA,MACjB,eAAe,SAAS;AAAA,MACxB,OAAO;AAAA,MACP,QAAQ,CAAC;AAAA,MACT;AAAA,MACA,kBAAkB,KAAK,IAAI;AAAA,IAC7B;AACA,SAAK,kBAAkB,YAAY,EAAE,IAAI;AACzC,SAAK,KAAK,oBAAoB,WAAW;AAAA,EAC3C;AAAA,EAEA,oBAAoB,OAA0C;AAC5D,UAAM,eAAe,MAAM;AAC3B,UAAM,aAAa,aAAa;AAChC,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,aAAS,SAAS,aAAa;AAC/B,aAAS,gBAAgB,aAAa;AACtC,aAAS,QAAQ,aAAa,SAAS;AACvC,SAAK,kBAAkB,UAAU,IAAI;AACrC,aAAS,QAAQ,QAAQ;AAEzB,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,SAAS,QAAQ;AAAA,MACvB,KAAK,UAAU;AACb,YAAI,SAAS,cAAc,SAAS,SAAU;AAC9C,cAAM,MAAM,SAAS,cAAc;AACnC,uBAAe,IAAI,sBAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B,MAAM,2BAAK;AAAA,UACX,SAAS,2BAAK;AAAA,QAChB,CAAC;AACD,aAAK,QACF,MAAM,EAAE,MAAM,2BAAK,MAAM,OAAO,2BAAK,QAAQ,CAAC,EAC9C,MAAM,4BAA4B;AACrC;AAAA,MACF;AAAA,MACA,KAAK,cAAc;AACjB,YAAI,SAAS,cAAc,SAAS,aAAc;AAClD,cAAM,SAAS,SAAS,cAAc;AACtC,uBAAe,IAAI,sBAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B;AAAA,QACF,CAAC;AACD,aAAK,QAAQ,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,gCAAgC;AACrE;AAAA,MACF;AAAA,MACA,KAAK,aAAa;AAChB,oBAAY;AACZ;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAK,iBAAiB,QAAQ;AAEnC,QAAI;AACJ,QAAI,SAAS,qBAAqB;AAChC,aAAO,SAAS,sBAAsB,SAAS;AAAA,IACjD;AACA,UAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AAEvC,UAAM,QAAQ,SAAS;AACvB,UAAM,SAAuC;AAAA,MAC3C,WAAW,SAAS;AAAA,MACpB,WAAW,SAAS;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,YAAY;AAAA,MACxB,mBAAkB,+BAAO,kBAAiB;AAAA,MAC1C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,mBAAmB,+BAAO,kBAAiB,KAAK,WAAY;AAAA,MAC5D,OAAO;AAAA,MACP,mBAAmB;AAAA,QACjB,eAAc,+BAAO,oBAAoB,kBAAiB;AAAA,QAC1D,aAAY,+BAAO,oBAAoB,gBAAe;AAAA,QACtD,cAAa,+BAAO,oBAAoB,iBAAgB;AAAA,MAC1D;AAAA,MACA,oBAAoB;AAAA,QAClB,aAAY,+BAAO,qBAAqB,gBAAe;AAAA,QACvD,cAAa,+BAAO,qBAAqB,iBAAgB;AAAA,MAC3D;AAAA,IACF;AACA,SAAK,KAAK,qBAAqB,MAAM;AAAA,EACvC;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,WAAW,MAAM;AAEvB,QAAI,SAAS,SAAS,aAAa,SAAS,SAAS,iBAAiB;AACpE,YAAM,IAAI,MAAM,yBAAyB,SAAS,IAAI,EAAE;AAAA,IAC1D;AAEA,QAAI;AACJ,QAAI,SAAS,SAAS,iBAAiB;AACrC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,SAAS;AAAA,IAClB;AAEA,UAAM,YAA4B;AAAA,MAChC;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,aAAa,MAAM;AAAA,MACnB,MAAM,SAAS;AAAA,MACf;AAAA,MACA,SAAS,CAAC;AAAA,MACV,SAAS,IAAI,qBAAO;AAAA,IACtB;AACA,yCAAU,OAAO,KAAK;AACtB,SAAK,KAAK,yBAAyB,SAAS;AAAA,EAC9C;AAAA,EAEA,8BAA8B,OAAoD;AAChF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,SAAI,iCAAQ,UAAS,iBAAiB;AACpC,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,QAAQ,MAAM,mDAAmD;AACtE;AAAA,MACF;AAGA,YAAM,OAAO,MAAM;AACnB,UAAI,KAAK,SAAS,iBAAiB;AACjC,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,YAAM,OAAO,KAAK,QAAQ,KAAK,IAAI;AACnC,UAAI,CAAC,MAAM;AACT,aAAK,QAAQ,MAAM,yBAAyB,KAAK,IAAI,YAAY;AACjE;AAAA,MACF;AAEA,WAAK,KAAK,yBAAyB;AAAA,QACjC,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,aAAa,KAAK,MAAM,KAAK,SAAS;AAE5C,WAAK,QAAQ;AAAA,QACX,kBAAkB,KAAK,OAAO,eAAe,KAAK,IAAI,mBAAmB,UAAU;AAAA,MACrF;AAEA,WAAK,QAAQ,UAAU,EAAE;AAAA,QACvB,CAAC,YAAY;AACX,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,aAAa,OAAO,EAAE;AACrF,eAAK,KAAK,2BAA2B;AAAA,YACnC,QAAQ,KAAK;AAAA,UACf,CAAC;AACD,eAAK,aAAa,KAAK;AAAA,YACrB,kBAAI,YAAY,6BAA6B;AAAA,cAC3C,MAAM,KAAK;AAAA,cACX,YAAY,KAAK;AAAA,cACjB,QAAQ;AAAA,YACV,CAAC;AAAA,YACD,OAAO;AAAA,UACT;AACA,eAAK,SAAS,OAAO;AAAA,QACvB;AAAA,QACA,CAAC,UAAU;AACT,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,gBAAgB,KAAK,EAAE;AAEtF,eAAK,KAAK,wBAAwB;AAAA,YAChC,QAAQ,KAAK;AAAA,UACf,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,qCAAQ,QAAQ;AAChB,SAAK,KAAK,wBAAwB,MAAM;AAAA,EAC1C;AAAA,EAEA,gCAAgC,OAAsD;AACpF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,UAAM,aAAa,IAAI,iCAA2B;AAClD,UAAM,cAAc,IAAI,iCAA+B;AAEvD,UAAM,aAA8B;AAAA,MAClC;AAAA,MACA,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,cAAc,MAAM;AAAA,MACpB,MAAM;AAAA,MACN,OAAO,CAAC;AAAA,MACR;AAAA,MACA;AAAA,MACA,WAAW,CAAC;AAAA,MACZ,aAAa,MAAM,KAAK;AAAA,IAC1B;AACA,qCAAQ,QAAQ,KAAK;AACrB,aAAU,sBAAsB,KAAK,IAAI;AACzC,SAAK,KAAK,0BAA0B,UAAU;AAAA,EAChD;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,SAAK,KAAK,yBAAyB,OAAO;AAAA,EAC5C;AAAA,EAEA,yBAAyB,OAA+C;AACtE,SAAK,KAAK,uBAAuB,KAAK;AAAA,EACxC;AAAA,EAEA,wBAAwB,OAA8C;AACpE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,OAAO,MAAM;AACrB,SAAK,KAAK,sBAAsB,KAAK;AAAA,EACvC;AAAA,EAEA,oCAAoC,OAA0D;AAC5F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,aAAa,MAAM;AACzB,YAAQ,QAAQ;AAEhB,YAAQ,WAAW,IAAI,UAAU;AAAA,EACnC;AAAA,EAEA,mCAAmC,OAAyD;AAC1F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,WAAW,MAAM;AAAA,EAC3B;AAAA,EAEA,0BAA0B,OAAgD;AACxE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO,QAAQ;AAC9C,UAAM,QAAQ,IAAI;AAAA,MAChB,IAAI,WAAW,KAAK,MAAM;AAAA,MAC1B,UAAU;AAAA,MACV,UAAU;AAAA,MACV,KAAK,SAAS;AAAA,IAChB;AACA,YAAQ,MAAM,KAAK,KAAK;AAExB,YAAQ,YAAY,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,yBAAyB,OAA+C;AACtE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,YAAY,MAAM;AAAA,EAC5B;AAAA,EAEA,0CAEE,OACM;AAAA,EAAC;AAAA,EAET,yCAEE,OACM;AAAA,EAAC;AAAA;AAAA,EAGT,yBAAyB,OAA+C;AAAA,EAAC;AAC3E;","names":["c"]}
+{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { metrics } from '@livekit/agents';\nimport {\n  type APIConnectOptions,\n  APIConnectionError,\n  APIError,\n  AudioByteStream,\n  DEFAULT_API_CONNECT_OPTIONS,\n  Future,\n  Queue,\n  Task,\n  cancelAndWait,\n  isAPIError,\n  llm,\n  log,\n  shortuuid,\n  stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport type { AudioResampler } from '@livekit/rtc-node';\nimport { AudioFrame, combineAudioFrames } from '@livekit/rtc-node';\nimport { delay } from '@std/async';\nimport type { GenerationCreatedEvent } from 'agents/dist/llm/realtime.js';\nimport { type MessageEvent, WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\nconst SAMPLE_RATE = 24000;\nconst NUM_CHANNELS = 1;\nconst BASE_URL = 'https://api.openai.com/v1';\n\nconst MOCK_AUDIO_ID_PREFIX = 'lk_mock_audio_item_';\n\ninterface RealtimeOptions {\n  model: api_proto.Model;\n  voice: api_proto.Voice;\n  temperature: number;\n  toolChoice?: llm.ToolChoice;\n  inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n  // TODO(shubhra): add inputAudioNoiseReduction\n  turnDetection?: api_proto.TurnDetectionType | null;\n  maxResponseOutputTokens?: number | 'inf';\n  speed?: number;\n  // TODO(shubhra): add openai tracing options\n  apiKey?: string;\n  baseURL: string;\n  isAzure: boolean;\n  azureDeployment?: string;\n  entraToken?: string;\n  apiVersion?: string;\n  maxSessionDuration: number;\n  // reset the connection after this many seconds if provided\n  connOptions: APIConnectOptions;\n}\n\ninterface MessageGeneration {\n  messageId: string;\n  textChannel: stream.StreamChannel<string>;\n  audioChannel: stream.StreamChannel<AudioFrame>;\n  audioTranscript: string;\n}\n\ninterface ResponseGeneration {\n  messageChannel: stream.StreamChannel<llm.MessageGeneration>;\n  functionChannel: stream.StreamChannel<llm.FunctionCall>;\n  messages: Map<string, MessageGeneration>;\n\n  /** @internal */\n  _doneFut: Future;\n  /** @internal */\n  _createdTimestamp: number;\n  /** @internal */\n  _firstTokenTimestamp?: number;\n}\n\nclass CreateResponseHandle {\n  instructions?: string;\n  doneFut: Future<llm.GenerationCreatedEvent>;\n  // TODO(shubhra): add timeout\n  constructor({ instructions }: { instructions?: string }) {\n    this.instructions = instructions;\n    this.doneFut = new Future();\n  }\n}\n\n// default values got from a \"default\" session from their API\nconst DEFAULT_FIRST_RETRY_INTERVAL_MS = 100;\nconst DEFAULT_TEMPERATURE = 0.8;\nconst DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n  type: 'server_vad',\n  threshold: 0.5,\n  prefix_padding_ms: 300,\n  silence_duration_ms: 200,\n  create_response: true,\n  interrupt_response: true,\n};\nconst DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n  model: 'gpt-4o-mini-transcribe',\n};\nconst DEFAULT_TOOL_CHOICE: llm.ToolChoice = 'auto';\nconst DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS: number | 'inf' = 'inf';\n\nconst AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n  model: 'whisper-1',\n};\n\nconst AZURE_DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n  type: 'server_vad',\n  threshold: 0.5,\n  prefix_padding_ms: 300,\n  silence_duration_ms: 200,\n  create_response: true,\n};\n\nconst DEFAULT_MAX_SESSION_DURATION = 20 * 60 * 1000; // 20 minutes\n\nconst DEFAULT_REALTIME_MODEL_OPTIONS = {\n  model: 'gpt-4o-realtime-preview',\n  voice: 'alloy',\n  temperature: DEFAULT_TEMPERATURE,\n  inputAudioTranscription: DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n  turnDetection: DEFAULT_TURN_DETECTION,\n  toolChoice: DEFAULT_TOOL_CHOICE,\n  maxResponseOutputTokens: DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS,\n  maxSessionDuration: DEFAULT_MAX_SESSION_DURATION,\n  connOptions: DEFAULT_API_CONNECT_OPTIONS,\n};\nexport class RealtimeModel extends llm.RealtimeModel {\n  sampleRate = api_proto.SAMPLE_RATE;\n  numChannels = api_proto.NUM_CHANNELS;\n  inFrameSize = api_proto.IN_FRAME_SIZE;\n  outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n  /* @internal */\n  _options: RealtimeOptions;\n\n  constructor(\n    options: {\n      model?: string;\n      voice?: string;\n      temperature?: number;\n      toolChoice?: llm.ToolChoice;\n      baseURL?: string;\n      inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n      // TODO(shubhra): add inputAudioNoiseReduction\n      turnDetection?: api_proto.TurnDetectionType | null;\n      speed?: number;\n      // TODO(shubhra): add openai tracing options\n      azureDeployment?: string;\n      apiKey?: string;\n      entraToken?: string;\n      apiVersion?: string;\n      maxSessionDuration?: number;\n      connOptions?: APIConnectOptions;\n    } = {},\n  ) {\n    super({\n      messageTruncation: true,\n      turnDetection: options.turnDetection !== null,\n      userTranscription: options.inputAudioTranscription !== null,\n      autoToolReplyGeneration: false,\n    });\n\n    const isAzure = !!(options.apiVersion || options.entraToken || options.azureDeployment);\n\n    if (options.apiKey === '' && !isAzure) {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable',\n      );\n    }\n\n    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;\n\n    if (!apiKey && !isAzure) {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable',\n      );\n    }\n\n    if (!options.baseURL && isAzure) {\n      const azureEndpoint = process.env.AZURE_OPENAI_ENDPOINT;\n      if (!azureEndpoint) {\n        throw new Error(\n          'Missing Azure endpoint. Please pass base_url or set AZURE_OPENAI_ENDPOINT environment variable.',\n        );\n      }\n      options.baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n    }\n\n    this._options = {\n      ...DEFAULT_REALTIME_MODEL_OPTIONS,\n      ...options,\n      baseURL: options.baseURL || BASE_URL,\n      apiKey,\n      isAzure,\n      model: options.model || DEFAULT_REALTIME_MODEL_OPTIONS.model,\n    };\n  }\n\n  /**\n   * Create a RealtimeModel instance configured for Azure OpenAI Service.\n   *\n   * @param azureDeployment - The name of your Azure OpenAI deployment.\n   * @param azureEndpoint - The endpoint URL for your Azure OpenAI resource. If undefined, will attempt to read from the environment variable AZURE_OPENAI_ENDPOINT.\n   * @param apiVersion - API version to use with Azure OpenAI Service. If undefined, will attempt to read from the environment variable OPENAI_API_VERSION.\n   * @param apiKey - Azure OpenAI API key. If undefined, will attempt to read from the environment variable AZURE_OPENAI_API_KEY.\n   * @param entraToken - Azure Entra authentication token. Required if not using API key authentication.\n   * @param baseURL - Base URL for the API endpoint. If undefined, constructed from the azure_endpoint.\n   * @param voice - Voice setting for audio outputs. Defaults to \"alloy\".\n   * @param inputAudioTranscription - Options for transcribing input audio. Defaults to @see DEFAULT_INPUT_AUDIO_TRANSCRIPTION.\n   * @param turnDetection - Options for server-based voice activity detection (VAD). Defaults to @see DEFAULT_SERVER_VAD_OPTIONS.\n   * @param temperature - Sampling temperature for response generation. Defaults to @see DEFAULT_TEMPERATURE.\n   * @param speed - Speed of the audio output. Defaults to 1.0.\n   * @param maxResponseOutputTokens - Maximum number of tokens in the response. Defaults to @see DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS.\n   * @param maxSessionDuration - Maximum duration of the session in milliseconds. Defaults to @see DEFAULT_MAX_SESSION_DURATION.\n   *\n   * @returns A RealtimeModel instance configured for Azure OpenAI Service.\n   *\n   * @throws Error if required Azure parameters are missing or invalid.\n   */\n  static withAzure({\n    azureDeployment,\n    azureEndpoint,\n    apiVersion,\n    apiKey,\n    entraToken,\n    baseURL,\n    voice = 'alloy',\n    inputAudioTranscription = AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n    turnDetection = AZURE_DEFAULT_TURN_DETECTION,\n    temperature = 0.8,\n    speed,\n  }: {\n    azureDeployment: string;\n    azureEndpoint?: string;\n    apiVersion?: string;\n    apiKey?: string;\n    entraToken?: string;\n    baseURL?: string;\n    voice?: string;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    // TODO(shubhra): add inputAudioNoiseReduction\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    speed?: number;\n  }) {\n    apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;\n    if (!apiKey && !entraToken) {\n      throw new Error(\n        'Missing credentials. Please pass one of `apiKey`, `entraToken`, or the `AZURE_OPENAI_API_KEY` environment variable.',\n      );\n    }\n\n    apiVersion = apiVersion || process.env.OPENAI_API_VERSION;\n    if (!apiVersion) {\n      throw new Error(\n        'Must provide either the `apiVersion` argument or the `OPENAI_API_VERSION` environment variable',\n      );\n    }\n\n    if (!baseURL) {\n      azureEndpoint = azureEndpoint || process.env.AZURE_OPENAI_ENDPOINT;\n      if (!azureEndpoint) {\n        throw new Error(\n          'Missing Azure endpoint. Please pass the `azure_endpoint` parameter or set the `AZURE_OPENAI_ENDPOINT` environment variable.',\n        );\n      }\n      baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n    }\n\n    return new RealtimeModel({\n      voice,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      speed,\n      apiKey,\n      azureDeployment,\n      apiVersion,\n      entraToken,\n      baseURL,\n    });\n  }\n\n  session() {\n    return new RealtimeSession(this);\n  }\n\n  async close() {\n    return;\n  }\n}\n\nfunction processBaseURL({\n  baseURL,\n  model,\n  isAzure = false,\n  azureDeployment,\n  apiVersion,\n}: {\n  baseURL: string;\n  model: string;\n  isAzure: boolean;\n  azureDeployment?: string;\n  apiVersion?: string;\n}): string {\n  const url = new URL([baseURL, 'realtime'].join('/'));\n\n  if (url.protocol === 'https:') {\n    url.protocol = 'wss:';\n  }\n\n  // ensure \"/realtime\" is added if the path is empty OR \"/v1\"\n  if (!url.pathname || ['', '/v1', '/openai'].includes(url.pathname.replace(/\\/$/, ''))) {\n    url.pathname = url.pathname.replace(/\\/$/, '') + '/realtime';\n  } else {\n    url.pathname = url.pathname.replace(/\\/$/, '');\n  }\n\n  const queryParams: Record<string, string> = {};\n  if (isAzure) {\n    if (apiVersion) {\n      queryParams['api-version'] = apiVersion;\n    }\n    if (azureDeployment) {\n      queryParams['deployment'] = azureDeployment;\n    }\n  } else {\n    queryParams['model'] = model;\n  }\n\n  for (const [key, value] of Object.entries(queryParams)) {\n    url.searchParams.set(key, value);\n  }\n\n  return url.toString();\n}\n\n/**\n * A session for the OpenAI Realtime API.\n *\n * This class is used to interact with the OpenAI Realtime API.\n * It is responsible for sending events to the OpenAI Realtime API and receiving events from it.\n *\n * It exposes two more events:\n * - openai_server_event_received: expose the raw server events from the OpenAI Realtime API\n * - openai_client_event_queued: expose the raw client events sent to the OpenAI Realtime API\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n  private _tools: llm.ToolContext = {};\n  private remoteChatCtx: llm.RemoteChatContext = new llm.RemoteChatContext();\n  private messageChannel = new Queue<api_proto.ClientEvent>();\n  private inputResampler?: AudioResampler;\n  private instructions?: string;\n  private oaiRealtimeModel: RealtimeModel;\n  private currentGeneration?: ResponseGeneration;\n  private responseCreatedFutures: { [id: string]: CreateResponseHandle } = {};\n\n  private textModeRecoveryRetries: number = 0;\n\n  private itemCreateFutures: { [id: string]: Future } = {};\n  private itemDeleteFutures: { [id: string]: Future } = {};\n\n  private updateChatCtxLock = new Mutex();\n  private updateFuncCtxLock = new Mutex();\n\n  // 100ms chunks\n  private bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS, SAMPLE_RATE / 10);\n\n  private pushedDurationMs: number = 0;\n\n  #logger = log();\n  #task: Promise<void>;\n  #closed = false;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super(realtimeModel);\n\n    this.oaiRealtimeModel = realtimeModel;\n\n    this.#task = this.#mainTask();\n\n    this.sendEvent(this.createSessionUpdateEvent());\n  }\n\n  sendEvent(command: api_proto.ClientEvent): void {\n    this.messageChannel.put(command);\n  }\n\n  private createSessionUpdateEvent(): api_proto.SessionUpdateEvent {\n    return {\n      type: 'session.update',\n      session: {\n        model: this.oaiRealtimeModel._options.model,\n        voice: this.oaiRealtimeModel._options.voice,\n        input_audio_format: 'pcm16',\n        output_audio_format: 'pcm16',\n        modalities: ['text', 'audio'],\n        turn_detection: this.oaiRealtimeModel._options.turnDetection,\n        input_audio_transcription: this.oaiRealtimeModel._options.inputAudioTranscription,\n        // TODO(shubhra): add inputAudioNoiseReduction\n        temperature: this.oaiRealtimeModel._options.temperature,\n        tool_choice: toOaiToolChoice(this.oaiRealtimeModel._options.toolChoice),\n        max_response_output_tokens:\n          this.oaiRealtimeModel._options.maxResponseOutputTokens === Infinity\n            ? 'inf'\n            : this.oaiRealtimeModel._options.maxResponseOutputTokens,\n        // TODO(shubhra): add tracing options\n        instructions: this.instructions,\n        speed: this.oaiRealtimeModel._options.speed,\n      },\n    };\n  }\n\n  get chatCtx() {\n    return this.remoteChatCtx.toChatCtx();\n  }\n\n  get tools() {\n    return { ...this._tools } as llm.ToolContext;\n  }\n\n  async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {\n    const unlock = await this.updateChatCtxLock.lock();\n    const events = this.createChatCtxUpdateEvents(_chatCtx);\n    const futures: Future<void>[] = [];\n\n    for (const event of events) {\n      const future = new Future<void>();\n      futures.push(future);\n\n      if (event.type === 'conversation.item.create') {\n        this.itemCreateFutures[event.item.id] = future;\n      } else if (event.type == 'conversation.item.delete') {\n        this.itemDeleteFutures[event.item_id] = future;\n      }\n\n      this.sendEvent(event);\n    }\n\n    if (futures.length === 0) {\n      unlock();\n      return;\n    }\n\n    try {\n      // wait for futures to resolve or timeout\n      await Promise.race([\n        Promise.all(futures),\n        delay(5000).then(() => {\n          throw new Error('Chat ctx update events timed out');\n        }),\n      ]);\n    } catch (e) {\n      this.#logger.error((e as Error).message);\n      throw e;\n    } finally {\n      unlock();\n    }\n  }\n\n  private createChatCtxUpdateEvents(\n    chatCtx: llm.ChatContext,\n    addMockAudio: boolean = false,\n  ): (api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[] {\n    const newChatCtx = chatCtx.copy();\n    if (addMockAudio) {\n      newChatCtx.items.push(createMockAudioItem());\n    } else {\n      // clean up existing mock audio items\n      newChatCtx.items = newChatCtx.items.filter(\n        (item) => !item.id.startsWith(MOCK_AUDIO_ID_PREFIX),\n      );\n    }\n\n    const events: (\n      | api_proto.ConversationItemCreateEvent\n      | api_proto.ConversationItemDeleteEvent\n    )[] = [];\n\n    const diffOps = llm.computeChatCtxDiff(this.chatCtx, newChatCtx);\n    for (const op of diffOps.toRemove) {\n      events.push({\n        type: 'conversation.item.delete',\n        item_id: op,\n        event_id: shortuuid('chat_ctx_delete_'),\n      } as api_proto.ConversationItemDeleteEvent);\n    }\n\n    for (const [previousId, id] of diffOps.toCreate) {\n      const chatItem = newChatCtx.getById(id);\n      if (!chatItem) {\n        throw new Error(`Chat item ${id} not found`);\n      }\n      events.push({\n        type: 'conversation.item.create',\n        item: livekitItemToOpenAIItem(chatItem),\n        previous_item_id: previousId ?? undefined,\n        event_id: shortuuid('chat_ctx_create_'),\n      } as api_proto.ConversationItemCreateEvent);\n    }\n    return events;\n  }\n\n  async updateTools(_tools: llm.ToolContext): Promise<void> {\n    const unlock = await this.updateFuncCtxLock.lock();\n    const ev = this.createToolsUpdateEvent(_tools);\n    this.sendEvent(ev);\n\n    if (!ev.session.tools) {\n      throw new Error('Tools are missing in the session update event');\n    }\n\n    // TODO(brian): these logics below are noops I think, leaving it here to keep\n    // parity with the python but we should remove them later\n    const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name));\n    const retainedTools = Object.fromEntries(\n      Object.entries(_tools).filter(\n        ([name, tool]) => llm.isFunctionTool(tool) && retainedToolNames.has(name),\n      ),\n    );\n\n    this._tools = retainedTools as llm.ToolContext;\n\n    unlock();\n  }\n\n  private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent {\n    const oaiTools: api_proto.Tool[] = [];\n\n    for (const [name, tool] of Object.entries(_tools)) {\n      if (!llm.isFunctionTool(tool)) {\n        this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n        continue;\n      }\n\n      const { parameters: toolParameters, description } = tool;\n      try {\n        const parameters = llm.toJsonSchema(\n          toolParameters,\n        ) as unknown as api_proto.Tool['parameters'];\n\n        oaiTools.push({\n          name,\n          description,\n          parameters: parameters,\n          type: 'function',\n        });\n      } catch (e) {\n        this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n        continue;\n      }\n    }\n\n    return {\n      type: 'session.update',\n      session: {\n        model: this.oaiRealtimeModel._options.model,\n        tools: oaiTools,\n      },\n      event_id: shortuuid('tools_update_'),\n    };\n  }\n\n  async updateInstructions(_instructions: string): Promise<void> {\n    const eventId = shortuuid('instructions_update_');\n    this.sendEvent({\n      type: 'session.update',\n      session: {\n        instructions: _instructions,\n      },\n      event_id: eventId,\n    } as api_proto.SessionUpdateEvent);\n    this.instructions = _instructions;\n  }\n\n  updateOptions({ toolChoice }: { toolChoice?: llm.ToolChoice }): void {\n    const options: api_proto.SessionUpdateEvent['session'] = {};\n\n    this.oaiRealtimeModel._options.toolChoice = toolChoice;\n    options.tool_choice = toOaiToolChoice(toolChoice);\n\n    // TODO(brian): add other options here\n\n    this.sendEvent({\n      type: 'session.update',\n      session: options,\n      event_id: shortuuid('options_update_'),\n    });\n  }\n\n  pushAudio(frame: AudioFrame): void {\n    for (const f of this.resampleAudio(frame)) {\n      for (const nf of this.bstream.write(f.data.buffer)) {\n        this.sendEvent({\n          type: 'input_audio_buffer.append',\n          audio: Buffer.from(nf.data.buffer).toString('base64'),\n        } as api_proto.InputAudioBufferAppendEvent);\n        // TODO(AJS-102): use frame.durationMs once available in rtc-node\n        this.pushedDurationMs += (nf.samplesPerChannel / nf.sampleRate) * 1000;\n      }\n    }\n  }\n\n  async commitAudio(): Promise<void> {\n    if (this.pushedDurationMs > 100) {\n      // OpenAI requires at least 100ms of audio\n      this.sendEvent({\n        type: 'input_audio_buffer.commit',\n      } as api_proto.InputAudioBufferCommitEvent);\n      this.pushedDurationMs = 0;\n    }\n  }\n\n  async clearAudio(): Promise<void> {\n    this.sendEvent({\n      type: 'input_audio_buffer.clear',\n    } as api_proto.InputAudioBufferClearEvent);\n    this.pushedDurationMs = 0;\n  }\n\n  async generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent> {\n    const handle = this.createResponse({ instructions, userInitiated: true });\n    this.textModeRecoveryRetries = 0;\n    return handle.doneFut.await;\n  }\n\n  async interrupt(): Promise<void> {\n    this.sendEvent({\n      type: 'response.cancel',\n    } as api_proto.ResponseCancelEvent);\n  }\n\n  async truncate(_options: { messageId: string; audioEndMs: number }): Promise<void> {\n    this.sendEvent({\n      type: 'conversation.item.truncate',\n      content_index: 0,\n      item_id: _options.messageId,\n      audio_end_ms: _options.audioEndMs,\n    } as api_proto.ConversationItemTruncateEvent);\n  }\n\n  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs\n  /// with large amounts of base64 audio data.\n  #loggableEvent(\n    event: api_proto.ClientEvent | api_proto.ServerEvent,\n    maxLength: number = 30,\n  ): Record<string, unknown> {\n    const untypedEvent: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(event)) {\n      if (value !== undefined) {\n        untypedEvent[key] = value;\n      }\n    }\n\n    if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n      const truncatedData =\n        untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? '…' : '');\n      return { ...untypedEvent, audio: truncatedData };\n    }\n    if (\n      untypedEvent.delta &&\n      typeof untypedEvent.delta === 'string' &&\n      event.type === 'response.audio.delta'\n    ) {\n      const truncatedDelta =\n        untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? '…' : '');\n      return { ...untypedEvent, delta: truncatedDelta };\n    }\n    return untypedEvent;\n  }\n\n  private async createWsConn(): Promise<WebSocket> {\n    const headers: Record<string, string> = {\n      'User-Agent': 'LiveKit-Agents-JS',\n    };\n\n    if (this.oaiRealtimeModel._options.isAzure) {\n      // Microsoft API has two ways of authentication\n      // 1. Entra token set as `Bearer` token\n      // 2. API key set as `api_key` header (also accepts query string)\n      if (this.oaiRealtimeModel._options.entraToken) {\n        headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.entraToken}`;\n      } else if (this.oaiRealtimeModel._options.apiKey) {\n        headers['api-key'] = this.oaiRealtimeModel._options.apiKey;\n      } else {\n        throw new Error('Microsoft API key or entraToken is required');\n      }\n    } else {\n      headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.apiKey}`;\n      headers['OpenAI-Beta'] = 'realtime=v1';\n    }\n\n    const url = processBaseURL({\n      baseURL: this.oaiRealtimeModel._options.baseURL,\n      model: this.oaiRealtimeModel._options.model,\n      isAzure: this.oaiRealtimeModel._options.isAzure,\n      apiVersion: this.oaiRealtimeModel._options.apiVersion,\n      azureDeployment: this.oaiRealtimeModel._options.azureDeployment,\n    });\n\n    this.#logger.debug(`Connecting to OpenAI Realtime API at ${url}`);\n\n    return new Promise((resolve, reject) => {\n      const ws = new WebSocket(url, { headers });\n      let waiting = true;\n\n      const timeout = setTimeout(() => {\n        ws.close();\n        reject(new Error('WebSocket connection timeout'));\n      }, this.oaiRealtimeModel._options.connOptions.timeoutMs);\n\n      ws.once('open', () => {\n        if (!waiting) return;\n        waiting = false;\n        clearTimeout(timeout);\n        resolve(ws);\n      });\n\n      ws.once('close', () => {\n        if (!waiting) return;\n        waiting = false;\n        clearTimeout(timeout);\n        reject(new Error('OpenAI Realtime API connection closed'));\n      });\n    });\n  }\n\n  async #mainTask(): Promise<void> {\n    let reconnecting = false;\n    let numRetries = 0;\n    let wsConn: WebSocket | null = null;\n    const maxRetries = this.oaiRealtimeModel._options.connOptions.maxRetry;\n\n    const reconnect = async () => {\n      this.#logger.debug(\n        {\n          maxSessionDuration: this.oaiRealtimeModel._options.maxSessionDuration,\n        },\n        'Reconnecting to OpenAI Realtime API',\n      );\n\n      const events: api_proto.ClientEvent[] = [];\n\n      // options and instructions\n      events.push(this.createSessionUpdateEvent());\n\n      // tools\n      if (Object.keys(this._tools).length > 0) {\n        events.push(this.createToolsUpdateEvent(this._tools));\n      }\n\n      // chat context\n      const chatCtx = this.chatCtx.copy({\n        excludeFunctionCall: true,\n        excludeInstructions: true,\n        excludeEmptyMessage: true,\n      });\n\n      const oldChatCtx = this.remoteChatCtx;\n      this.remoteChatCtx = new llm.RemoteChatContext();\n      events.push(...this.createChatCtxUpdateEvents(chatCtx));\n\n      try {\n        for (const ev of events) {\n          this.emit('openai_client_event_queued', ev);\n          wsConn!.send(JSON.stringify(ev));\n        }\n      } catch (error) {\n        this.remoteChatCtx = oldChatCtx;\n        throw new APIConnectionError({\n          message: 'Failed to send message to OpenAI Realtime API during session re-connection',\n        });\n      }\n\n      this.#logger.debug('Reconnected to OpenAI Realtime API');\n\n      this.emit('session_reconnected', {} as llm.RealtimeSessionReconnectedEvent);\n    };\n\n    reconnecting = false;\n    while (!this.#closed) {\n      this.#logger.debug('Creating WebSocket connection to OpenAI Realtime API');\n      wsConn = await this.createWsConn();\n\n      try {\n        if (reconnecting) {\n          await reconnect();\n          numRetries = 0;\n        }\n        await this.runWs(wsConn);\n      } catch (error) {\n        if (!isAPIError(error)) {\n          this.emitError({ error: error as Error, recoverable: false });\n          throw error;\n        }\n\n        if (maxRetries === 0 || !error.retryable) {\n          this.emitError({ error: error as Error, recoverable: false });\n          throw error;\n        }\n\n        if (numRetries === maxRetries) {\n          this.emitError({ error: error as Error, recoverable: false });\n          throw new APIConnectionError({\n            message: `OpenAI Realtime API connection failed after ${numRetries} attempts`,\n            options: {\n              body: error,\n              retryable: false,\n            },\n          });\n        }\n\n        this.emitError({ error: error as Error, recoverable: true });\n        const retryInterval =\n          numRetries === 0\n            ? DEFAULT_FIRST_RETRY_INTERVAL_MS\n            : this.oaiRealtimeModel._options.connOptions.retryIntervalMs;\n        this.#logger.warn(\n          {\n            attempt: numRetries,\n            maxRetries,\n            error,\n          },\n          `OpenAI Realtime API connection failed, retrying in ${retryInterval / 1000}s`,\n        );\n\n        await delay(retryInterval);\n        numRetries++;\n      }\n\n      reconnecting = true;\n    }\n  }\n\n  private async runWs(wsConn: WebSocket): Promise<void> {\n    const forwardEvents = async (signal: AbortSignal): Promise<void> => {\n      while (!this.#closed && wsConn.readyState === WebSocket.OPEN && !signal.aborted) {\n        try {\n          const event = await this.messageChannel.get();\n          if (signal.aborted) {\n            break;\n          }\n\n          if (event.type !== 'input_audio_buffer.append') {\n            this.#logger.debug(`(client) -> ${JSON.stringify(this.#loggableEvent(event))}`);\n          }\n\n          this.emit('openai_client_event_queued', event);\n          wsConn.send(JSON.stringify(event));\n        } catch (error) {\n          break;\n        }\n      }\n\n      wsConn.close();\n    };\n\n    const wsCloseFuture = new Future<void | Error>();\n\n    wsConn.onerror = (error) => {\n      wsCloseFuture.resolve(new APIConnectionError({ message: error.message }));\n    };\n    wsConn.onclose = () => {\n      wsCloseFuture.resolve();\n    };\n\n    wsConn.onmessage = (message: MessageEvent) => {\n      const event: api_proto.ServerEvent = JSON.parse(message.data as string);\n\n      this.emit('openai_server_event_received', event);\n      this.#logger.debug(`(server) <- ${JSON.stringify(this.#loggableEvent(event))}`);\n\n      switch (event.type) {\n        case 'input_audio_buffer.speech_started':\n          this.handleInputAudioBufferSpeechStarted(event);\n          break;\n        case 'input_audio_buffer.speech_stopped':\n          this.handleInputAudioBufferSpeechStopped(event);\n          break;\n        case 'response.created':\n          this.handleResponseCreated(event);\n          break;\n        case 'response.output_item.added':\n          this.handleResponseOutputItemAdded(event);\n          break;\n        case 'conversation.item.created':\n          this.handleConversationItemCreated(event);\n          break;\n        case 'conversation.item.deleted':\n          this.handleConversationItemDeleted(event);\n          break;\n        case 'conversation.item.input_audio_transcription.completed':\n          this.handleConversationItemInputAudioTranscriptionCompleted(event);\n          break;\n        case 'conversation.item.input_audio_transcription.failed':\n          this.handleConversationItemInputAudioTranscriptionFailed(event);\n          break;\n        case 'response.content_part.added':\n          this.handleResponseContentPartAdded(event);\n          break;\n        case 'response.content_part.done':\n          this.handleResponseContentPartDone(event);\n          break;\n        case 'response.audio_transcript.delta':\n          this.handleResponseAudioTranscriptDelta(event);\n          break;\n        case 'response.audio.delta':\n          this.handleResponseAudioDelta(event);\n          break;\n        case 'response.audio_transcript.done':\n          this.handleResponseAudioTranscriptDone(event);\n          break;\n        case 'response.audio.done':\n          this.handleResponseAudioDone(event);\n          break;\n        case 'response.output_item.done':\n          this.handleResponseOutputItemDone(event);\n          break;\n        case 'response.done':\n          this.handleResponseDone(event);\n          break;\n        case 'error':\n          this.handleError(event);\n          break;\n        default:\n          this.#logger.debug(`unhandled event: ${event.type}`);\n          break;\n      }\n    };\n\n    const sendTask = Task.from(({ signal }) => forwardEvents(signal));\n\n    const wsTask = Task.from(({ signal }) => {\n      const abortPromise = new Promise<void>((resolve) => {\n        signal.addEventListener('abort', () => {\n          resolve();\n        });\n      });\n\n      return Promise.race([wsCloseFuture.await, abortPromise]);\n    });\n\n    const waitReconnectTask = Task.from(async ({ signal }) => {\n      await delay(this.oaiRealtimeModel._options.maxSessionDuration, { signal });\n      return new APIConnectionError({\n        message: 'OpenAI Realtime API connection timeout',\n      });\n    });\n\n    try {\n      const result = await Promise.race([wsTask.result, sendTask.result, waitReconnectTask.result]);\n\n      if (waitReconnectTask.done && this.currentGeneration) {\n        await this.currentGeneration._doneFut.await;\n      }\n\n      if (result instanceof Error) {\n        throw result;\n      }\n    } finally {\n      await cancelAndWait([wsTask, sendTask, waitReconnectTask], 2000);\n      wsConn.close();\n    }\n  }\n\n  async close() {\n    super.close();\n    this.#closed = true;\n    await this.#task;\n  }\n\n  private handleInputAudioBufferSpeechStarted(\n    _event: api_proto.InputAudioBufferSpeechStartedEvent,\n  ): void {\n    this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n  }\n\n  private handleInputAudioBufferSpeechStopped(\n    _event: api_proto.InputAudioBufferSpeechStoppedEvent,\n  ): void {\n    this.emit('input_speech_stopped', {\n      userTranscriptionEnabled: this.oaiRealtimeModel._options.inputAudioTranscription !== null,\n    } as llm.InputSpeechStoppedEvent);\n  }\n\n  private handleResponseCreated(event: api_proto.ResponseCreatedEvent): void {\n    if (!event.response.id) {\n      throw new Error('response.id is missing');\n    }\n\n    this.currentGeneration = {\n      messageChannel: stream.createStreamChannel<llm.MessageGeneration>(),\n      functionChannel: stream.createStreamChannel<llm.FunctionCall>(),\n      messages: new Map(),\n      _doneFut: new Future(),\n      _createdTimestamp: Date.now(),\n    };\n\n    if (!event.response.metadata || !event.response.metadata.client_event_id) return;\n\n    const handle = this.responseCreatedFutures[event.response.metadata.client_event_id];\n    if (handle) {\n      delete this.responseCreatedFutures[event.response.metadata.client_event_id];\n\n      // set key to the response id\n      this.responseCreatedFutures[event.response.id] = handle;\n    }\n\n    // the generation_created event is emitted when\n    // 1. the response is not a message on response.output_item.added event\n    // 2. the content is audio on response.content_part.added event\n    // will try to recover from text response on response.content_part.done event\n    this.emit('generation_created', {\n      messageStream: this.currentGeneration.messageChannel.stream(),\n      functionStream: this.currentGeneration.functionChannel.stream(),\n      userInitiated: false,\n    } as GenerationCreatedEvent);\n  }\n\n  private handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    if (!event.item.type) {\n      throw new Error('item.type is not set');\n    }\n\n    if (!event.response_id) {\n      throw new Error('response_id is not set');\n    }\n\n    const itemType = event.item.type;\n    const responseId = event.response_id;\n\n    if (itemType !== 'message') {\n      // emit immediately if it's not a message, otherwise wait response.content_part.added\n      this.emitGenerationEvent(responseId);\n      this.textModeRecoveryRetries = 0;\n      return;\n    }\n  }\n\n  private handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {\n    if (!event.item.id) {\n      throw new Error('item.id is not set');\n    }\n\n    try {\n      this.remoteChatCtx.insert(event.previous_item_id, openAIItemToLivekitItem(event.item));\n    } catch (error) {\n      this.#logger.error({ error, itemId: event.item.id }, 'failed to insert conversation item');\n    }\n\n    const fut = this.itemCreateFutures[event.item.id];\n    if (fut) {\n      fut.resolve();\n      delete this.itemCreateFutures[event.item.id];\n    }\n  }\n\n  private handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {\n    if (!event.item_id) {\n      throw new Error('item_id is not set');\n    }\n\n    try {\n      this.remoteChatCtx.delete(event.item_id);\n    } catch (error) {\n      this.#logger.error({ error, itemId: event.item_id }, 'failed to delete conversation item');\n    }\n\n    const fut = this.itemDeleteFutures[event.item_id];\n    if (fut) {\n      fut.resolve();\n      delete this.itemDeleteFutures[event.item_id];\n    }\n  }\n\n  private handleConversationItemInputAudioTranscriptionCompleted(\n    event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n  ): void {\n    const remoteItem = this.remoteChatCtx.get(event.item_id);\n    if (!remoteItem) {\n      return;\n    }\n\n    const item = remoteItem.item;\n    if (item instanceof llm.ChatMessage) {\n      item.content.push(event.transcript);\n    } else {\n      throw new Error('item is not a chat message');\n    }\n\n    this.emit('input_audio_transcription_completed', {\n      itemId: event.item_id,\n      transcript: event.transcript,\n      isFinal: true,\n    } as llm.InputTranscriptionCompleted);\n  }\n\n  private handleConversationItemInputAudioTranscriptionFailed(\n    event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n  ): void {\n    this.#logger.error(\n      { error: event.error },\n      'OpenAI Realtime API failed to transcribe input audio',\n    );\n  }\n\n  private handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemId = event.item_id;\n    const itemType = event.part.type;\n    const responseId = event.response_id;\n\n    if (itemType === 'audio') {\n      this.emitGenerationEvent(responseId);\n      if (this.textModeRecoveryRetries > 0) {\n        this.#logger.info(\n          { retries: this.textModeRecoveryRetries },\n          'recovered from text-only response',\n        );\n        this.textModeRecoveryRetries = 0;\n      }\n\n      const itemGeneration: MessageGeneration = {\n        messageId: itemId,\n        textChannel: stream.createStreamChannel<string>(),\n        audioChannel: stream.createStreamChannel<AudioFrame>(),\n        audioTranscript: '',\n      };\n\n      this.currentGeneration.messageChannel.write({\n        messageId: itemId,\n        textStream: itemGeneration.textChannel.stream(),\n        audioStream: itemGeneration.audioChannel.stream(),\n      });\n\n      this.currentGeneration.messages.set(itemId, itemGeneration);\n      this.currentGeneration._firstTokenTimestamp = Date.now();\n      return;\n    } else {\n      this.interrupt();\n      if (this.textModeRecoveryRetries === 0) {\n        this.#logger.warn({ responseId }, 'received text-only response from OpenAI Realtime API');\n      }\n    }\n  }\n\n  private handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n    if (event.part.type !== 'text') {\n      return;\n    }\n\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    // TODO(shubhra): handle text mode recovery\n  }\n\n  private handleResponseAudioTranscriptDelta(\n    event: api_proto.ResponseAudioTranscriptDeltaEvent,\n  ): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemId = event.item_id;\n    const delta = event.delta;\n\n    // TODO (shubhra): add timed string support\n\n    const itemGeneration = this.currentGeneration.messages.get(itemId);\n    if (!itemGeneration) {\n      throw new Error('itemGeneration is not set');\n    } else {\n      itemGeneration.textChannel.write(delta);\n      itemGeneration.audioTranscript += delta;\n    }\n  }\n\n  private handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemGeneration = this.currentGeneration.messages.get(event.item_id);\n    if (!itemGeneration) {\n      throw new Error('itemGeneration is not set');\n    }\n\n    const binaryString = atob(event.delta);\n    const len = binaryString.length;\n    const bytes = new Uint8Array(len);\n    for (let i = 0; i < len; i++) {\n      bytes[i] = binaryString.charCodeAt(i);\n    }\n\n    itemGeneration.audioChannel.write(\n      new AudioFrame(\n        new Int16Array(bytes.buffer),\n        api_proto.SAMPLE_RATE,\n        api_proto.NUM_CHANNELS,\n        bytes.length / 2,\n      ),\n    );\n  }\n\n  private handleResponseAudioTranscriptDone(\n    _event: api_proto.ResponseAudioTranscriptDoneEvent,\n  ): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n  }\n\n  private handleResponseAudioDone(_event: api_proto.ResponseAudioDoneEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n  }\n\n  private handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemId = event.item.id;\n    const itemType = event.item.type;\n\n    if (itemType === 'function_call') {\n      const item = event.item;\n      if (!item.call_id || !item.name || !item.arguments) {\n        throw new Error('item is not a function call');\n      }\n      this.currentGeneration.functionChannel.write({\n        callId: item.call_id,\n        name: item.name,\n        args: item.arguments,\n      } as llm.FunctionCall);\n    } else if (itemType === 'message') {\n      const itemGeneration = this.currentGeneration.messages.get(itemId);\n      if (!itemGeneration) {\n        return;\n      }\n      // text response doesn't have itemGeneration\n      itemGeneration.textChannel.close();\n      itemGeneration.audioChannel.close();\n    }\n  }\n\n  private handleResponseDone(_event: api_proto.ResponseDoneEvent): void {\n    if (!this.currentGeneration) {\n      // OpenAI has a race condition where we could receive response.done without any\n      // previous response.created (This happens generally during interruption)\n      return;\n    }\n\n    const createdTimestamp = this.currentGeneration._createdTimestamp;\n    const firstTokenTimestamp = this.currentGeneration._firstTokenTimestamp;\n\n    this.#logger.debug(\n      {\n        messageCount: this.currentGeneration.messages.size,\n      },\n      'Closing generation channels in handleResponseDone',\n    );\n\n    for (const generation of this.currentGeneration.messages.values()) {\n      generation.textChannel.close();\n      generation.audioChannel.close();\n    }\n\n    this.currentGeneration.functionChannel.close();\n    this.currentGeneration.messageChannel.close();\n\n    for (const itemId of this.currentGeneration.messages.keys()) {\n      const remoteItem = this.remoteChatCtx.get(itemId);\n      if (remoteItem && remoteItem.item instanceof llm.ChatMessage) {\n        remoteItem.item.content.push(this.currentGeneration.messages.get(itemId)!.audioTranscript);\n      }\n    }\n\n    this.currentGeneration._doneFut.resolve();\n    this.currentGeneration = undefined;\n\n    // Calculate and emit metrics\n    const usage = _event.response.usage;\n    const ttft = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n    const duration = (Date.now() - createdTimestamp) / 1000; // Convert to seconds\n\n    const realtimeMetrics: metrics.RealtimeModelMetrics = {\n      type: 'realtime_model_metrics',\n      timestamp: createdTimestamp / 1000, // Convert to seconds\n      requestId: _event.response.id || '',\n      ttft,\n      duration,\n      cancelled: _event.response.status === 'cancelled',\n      label: 'openai_realtime',\n      inputTokens: usage?.input_tokens ?? 0,\n      outputTokens: usage?.output_tokens ?? 0,\n      totalTokens: usage?.total_tokens ?? 0,\n      tokensPerSecond: duration > 0 ? (usage?.output_tokens ?? 0) / duration : 0,\n      inputTokenDetails: {\n        audioTokens: usage?.input_token_details?.audio_tokens ?? 0,\n        textTokens: usage?.input_token_details?.text_tokens ?? 0,\n        imageTokens: 0, // Not supported yet\n        cachedTokens: usage?.input_token_details?.cached_tokens ?? 0,\n        cachedTokensDetails: usage?.input_token_details?.cached_tokens_details\n          ? {\n              audioTokens: usage?.input_token_details?.cached_tokens_details?.audio_tokens ?? 0,\n              textTokens: usage?.input_token_details?.cached_tokens_details?.text_tokens ?? 0,\n              imageTokens: usage?.input_token_details?.cached_tokens_details?.image_tokens ?? 0,\n            }\n          : undefined,\n      },\n      outputTokenDetails: {\n        textTokens: usage?.output_token_details?.text_tokens ?? 0,\n        audioTokens: usage?.output_token_details?.audio_tokens ?? 0,\n        imageTokens: 0,\n      },\n    };\n\n    this.emit('metrics_collected', realtimeMetrics);\n    // TODO(brian): handle response done but not complete\n  }\n\n  private handleError(event: api_proto.ErrorEvent): void {\n    if (event.error.message.startsWith('Cancellation failed')) {\n      return;\n    }\n\n    this.#logger.error({ error: event.error }, 'OpenAI Realtime API returned an error');\n    this.emitError({\n      error: new APIError(event.error.message, {\n        body: event.error,\n        retryable: true,\n      }),\n      recoverable: true,\n    });\n\n    // TODO(brian): set error for response future if it exists\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }): void {\n    // IMPORTANT: only emit error if there are listeners; otherwise emit will throw an error\n    this.emit('error', {\n      timestamp: Date.now(),\n      // TODO(brian): add label\n      label: '',\n      error,\n      recoverable,\n    } as llm.RealtimeModelError);\n  }\n\n  private *resampleAudio(frame: AudioFrame): Generator<AudioFrame> {\n    yield frame;\n  }\n\n  private createResponse({\n    userInitiated,\n    instructions,\n    oldHandle,\n  }: {\n    userInitiated: boolean;\n    instructions?: string;\n    oldHandle?: CreateResponseHandle;\n  }): CreateResponseHandle {\n    const handle = oldHandle || new CreateResponseHandle({ instructions });\n    if (oldHandle && instructions) {\n      handle.instructions = instructions;\n    }\n\n    const eventId = shortuuid('response_create_');\n    if (userInitiated) {\n      this.responseCreatedFutures[eventId] = handle;\n    }\n\n    const response: api_proto.ResponseCreateEvent['response'] = {};\n    if (instructions) response.instructions = instructions;\n    if (userInitiated) response.metadata = { client_event_id: eventId };\n\n    this.sendEvent({\n      type: 'response.create',\n      event_id: eventId,\n      response: Object.keys(response).length > 0 ? response : undefined,\n    });\n\n    return handle;\n  }\n\n  private emitGenerationEvent(responseId: string): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const generation_ev: llm.GenerationCreatedEvent = {\n      messageStream: this.currentGeneration.messageChannel.stream(),\n      functionStream: this.currentGeneration.functionChannel.stream(),\n      userInitiated: false,\n    };\n\n    const handle = this.responseCreatedFutures[responseId];\n    if (handle) {\n      delete this.responseCreatedFutures[responseId];\n      generation_ev.userInitiated = true;\n      if (handle.doneFut.done) {\n        this.#logger.warn({ responseId }, 'response received after timeout');\n      } else {\n        handle.doneFut.resolve(generation_ev);\n      }\n    }\n\n    this.#logger.debug({ responseId }, 'Emitting generation_created event');\n    this.emit('generation_created', generation_ev);\n  }\n}\n\nfunction livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {\n  switch (item.type) {\n    case 'function_call':\n      return {\n        id: item.id,\n        type: 'function_call',\n        call_id: item.callId,\n        name: item.name,\n        arguments: item.args,\n      } as api_proto.FunctionCallItem;\n    case 'function_call_output':\n      return {\n        id: item.id,\n        type: 'function_call_output',\n        call_id: item.callId,\n        output: item.output,\n      } as api_proto.FunctionCallOutputItem;\n    case 'message':\n      const role = item.role === 'developer' ? 'system' : item.role;\n      const contentList: api_proto.Content[] = [];\n      for (const c of item.content) {\n        if (typeof c === 'string') {\n          contentList.push({\n            type: role === 'assistant' ? 'text' : 'input_text',\n            text: c,\n          } as api_proto.InputTextContent);\n        } else if (c.type === 'image_content') {\n          // not supported for now\n          continue;\n        } else if (c.type === 'audio_content') {\n          if (role === 'user') {\n            const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString('base64');\n            contentList.push({\n              type: 'input_audio',\n              audio: encodedAudio,\n            } as api_proto.InputAudioContent);\n          }\n        }\n      }\n      return {\n        id: item.id,\n        type: 'message',\n        role,\n        content: contentList,\n      } as api_proto.UserItem;\n  }\n}\n\nfunction openAIItemToLivekitItem(item: api_proto.ItemResource): llm.ChatItem {\n  if (!item.id) {\n    throw new Error('item.id is not set');\n  }\n\n  switch (item.type) {\n    case 'function_call':\n      return llm.FunctionCall.create({\n        id: item.id,\n        callId: item.call_id,\n        name: item.name,\n        args: item.arguments,\n      });\n    case 'function_call_output':\n      return llm.FunctionCallOutput.create({\n        id: item.id,\n        callId: item.call_id,\n        output: item.output,\n        isError: false,\n      });\n    case 'message':\n      const content: llm.ChatContent[] = [];\n      // item.content can be a single object or an array; normalize to array\n      const contents = Array.isArray(item.content) ? item.content : [item.content];\n      for (const c of contents) {\n        if (c.type === 'text' || c.type === 'input_text') {\n          content.push(c.text);\n        }\n      }\n      return llm.ChatMessage.create({\n        id: item.id,\n        role: item.role,\n        content,\n      });\n  }\n}\n\nfunction createMockAudioItem(durationSeconds: number = 2): llm.ChatMessage {\n  const audioData = Buffer.alloc(durationSeconds * SAMPLE_RATE);\n  return llm.ChatMessage.create({\n    id: shortuuid(MOCK_AUDIO_ID_PREFIX),\n    role: 'user',\n    content: [\n      {\n        type: 'audio_content',\n        frame: [\n          new AudioFrame(\n            new Int16Array(audioData.buffer),\n            SAMPLE_RATE,\n            NUM_CHANNELS,\n            audioData.length / 2,\n          ),\n        ],\n      } as llm.AudioContent,\n    ],\n  });\n}\n\nfunction toOaiToolChoice(toolChoice?: llm.ToolChoice): api_proto.ToolChoice {\n  if (typeof toolChoice === 'string') {\n    return toolChoice;\n  }\n\n  if (toolChoice?.type === 'function') {\n    return toolChoice.function.name;\n  }\n\n  return 'auto';\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAeO;AACP,mBAAsB;AAEtB,sBAA+C;AAC/C,mBAAsB;AAEtB,gBAA6C;AAC7C,gBAA2B;AAE3B,MAAM,cAAc;AACpB,MAAM,eAAe;AACrB,MAAM,WAAW;AAEjB,MAAM,uBAAuB;AA4C7B,MAAM,qBAAqB;AAAA,EACzB;AAAA,EACA;AAAA;AAAA,EAEA,YAAY,EAAE,aAAa,GAA8B;AACvD,SAAK,eAAe;AACpB,SAAK,UAAU,IAAI,qBAAO;AAAA,EAC5B;AACF;AAGA,MAAM,kCAAkC;AACxC,MAAM,sBAAsB;AAC5B,MAAM,yBAAsD;AAAA,EAC1D,MAAM;AAAA,EACN,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,iBAAiB;AAAA,EACjB,oBAAoB;AACtB;AACA,MAAM,oCAAuE;AAAA,EAC3E,OAAO;AACT;AACA,MAAM,sBAAsC;AAC5C,MAAM,qCAAqD;AAE3D,MAAM,0CAA6E;AAAA,EACjF,OAAO;AACT;AAEA,MAAM,+BAA4D;AAAA,EAChE,MAAM;AAAA,EACN,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,iBAAiB;AACnB;AAEA,MAAM,+BAA+B,KAAK,KAAK;AAE/C,MAAM,iCAAiC;AAAA,EACrC,OAAO;AAAA,EACP,OAAO;AAAA,EACP,aAAa;AAAA,EACb,yBAAyB;AAAA,EACzB,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,yBAAyB;AAAA,EACzB,oBAAoB;AAAA,EACpB,aAAa;AACf;AACO,MAAM,sBAAsB,kBAAI,cAAc;AAAA,EACnD,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA;AAAA,EAGzB;AAAA,EAEA,YACE,UAiBI,CAAC,GACL;AACA,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe,QAAQ,kBAAkB;AAAA,MACzC,mBAAmB,QAAQ,4BAA4B;AAAA,MACvD,yBAAyB;AAAA,IAC3B,CAAC;AAED,UAAM,UAAU,CAAC,EAAE,QAAQ,cAAc,QAAQ,cAAc,QAAQ;AAEvE,QAAI,QAAQ,WAAW,MAAM,CAAC,SAAS;AACrC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAE7C,QAAI,CAAC,UAAU,CAAC,SAAS;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,QAAQ,WAAW,SAAS;AAC/B,YAAM,gBAAgB,QAAQ,IAAI;AAClC,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,cAAQ,UAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IACvD;AAEA,SAAK,WAAW;AAAA,MACd,GAAG;AAAA,MACH,GAAG;AAAA,MACH,SAAS,QAAQ,WAAW;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,OAAO,QAAQ,SAAS,+BAA+B;AAAA,IACzD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuBA,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR,0BAA0B;AAAA,IAC1B,gBAAgB;AAAA,IAChB,cAAc;AAAA,IACd;AAAA,EACF,GAaG;AACD,aAAS,UAAU,QAAQ,IAAI;AAC/B,QAAI,CAAC,UAAU,CAAC,YAAY;AAC1B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,iBAAa,cAAc,QAAQ,IAAI;AACvC,QAAI,CAAC,YAAY;AACf,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,SAAS;AACZ,sBAAgB,iBAAiB,QAAQ,IAAI;AAC7C,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,gBAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IAC/C;AAEA,WAAO,IAAI,cAAc;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA,EAEA,MAAM,QAAQ;AACZ;AAAA,EACF;AACF;AAEA,SAAS,eAAe;AAAA,EACtB;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV;AAAA,EACA;AACF,GAMW;AACT,QAAM,MAAM,IAAI,IAAI,CAAC,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAEnD,MAAI,IAAI,aAAa,UAAU;AAC7B,QAAI,WAAW;AAAA,EACjB;AAGA,MAAI,CAAC,IAAI,YAAY,CAAC,IAAI,OAAO,SAAS,EAAE,SAAS,IAAI,SAAS,QAAQ,OAAO,EAAE,CAAC,GAAG;AACrF,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE,IAAI;AAAA,EACnD,OAAO;AACL,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE;AAAA,EAC/C;AAEA,QAAM,cAAsC,CAAC;AAC7C,MAAI,SAAS;AACX,QAAI,YAAY;AACd,kBAAY,aAAa,IAAI;AAAA,IAC/B;AACA,QAAI,iBAAiB;AACnB,kBAAY,YAAY,IAAI;AAAA,IAC9B;AAAA,EACF,OAAO;AACL,gBAAY,OAAO,IAAI;AAAA,EACzB;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,QAAI,aAAa,IAAI,KAAK,KAAK;AAAA,EACjC;AAEA,SAAO,IAAI,SAAS;AACtB;AAYO,MAAM,wBAAwB,kBAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,gBAAuC,IAAI,kBAAI,kBAAkB;AAAA,EACjE,iBAAiB,IAAI,oBAA6B;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAiE,CAAC;AAAA,EAElE,0BAAkC;AAAA,EAElC,oBAA8C,CAAC;AAAA,EAC/C,oBAA8C,CAAC;AAAA,EAE/C,oBAAoB,IAAI,mBAAM;AAAA,EAC9B,oBAAoB,IAAI,mBAAM;AAAA;AAAA,EAG9B,UAAU,IAAI,8BAAgB,aAAa,cAAc,cAAc,EAAE;AAAA,EAEzE,mBAA2B;AAAA,EAEnC,cAAU,mBAAI;AAAA,EACd;AAAA,EACA,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,mBAAmB;AAExB,SAAK,QAAQ,KAAK,UAAU;AAE5B,SAAK,UAAU,KAAK,yBAAyB,CAAC;AAAA,EAChD;AAAA,EAEA,UAAU,SAAsC;AAC9C,SAAK,eAAe,IAAI,OAAO;AAAA,EACjC;AAAA,EAEQ,2BAAyD;AAC/D,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,oBAAoB;AAAA,QACpB,qBAAqB;AAAA,QACrB,YAAY,CAAC,QAAQ,OAAO;AAAA,QAC5B,gBAAgB,KAAK,iBAAiB,SAAS;AAAA,QAC/C,2BAA2B,KAAK,iBAAiB,SAAS;AAAA;AAAA,QAE1D,aAAa,KAAK,iBAAiB,SAAS;AAAA,QAC5C,aAAa,gBAAgB,KAAK,iBAAiB,SAAS,UAAU;AAAA,QACtE,4BACE,KAAK,iBAAiB,SAAS,4BAA4B,WACvD,QACA,KAAK,iBAAiB,SAAS;AAAA;AAAA,QAErC,cAAc,KAAK;AAAA,QACnB,OAAO,KAAK,iBAAiB,SAAS;AAAA,MACxC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,UAAU;AACZ,WAAO,KAAK,cAAc,UAAU;AAAA,EACtC;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,UAA0C;AAC5D,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,SAAS,KAAK,0BAA0B,QAAQ;AACtD,UAAM,UAA0B,CAAC;AAEjC,eAAW,SAAS,QAAQ;AAC1B,YAAM,SAAS,IAAI,qBAAa;AAChC,cAAQ,KAAK,MAAM;AAEnB,UAAI,MAAM,SAAS,4BAA4B;AAC7C,aAAK,kBAAkB,MAAM,KAAK,EAAE,IAAI;AAAA,MAC1C,WAAW,MAAM,QAAQ,4BAA4B;AACnD,aAAK,kBAAkB,MAAM,OAAO,IAAI;AAAA,MAC1C;AAEA,WAAK,UAAU,KAAK;AAAA,IACtB;AAEA,QAAI,QAAQ,WAAW,GAAG;AACxB,aAAO;AACP;AAAA,IACF;AAEA,QAAI;AAEF,YAAM,QAAQ,KAAK;AAAA,QACjB,QAAQ,IAAI,OAAO;AAAA,YACnB,oBAAM,GAAI,EAAE,KAAK,MAAM;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACpD,CAAC;AAAA,MACH,CAAC;AAAA,IACH,SAAS,GAAG;AACV,WAAK,QAAQ,MAAO,EAAY,OAAO;AACvC,YAAM;AAAA,IACR,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEQ,0BACN,SACA,eAAwB,OAC2D;AACnF,UAAM,aAAa,QAAQ,KAAK;AAChC,QAAI,cAAc;AAChB,iBAAW,MAAM,KAAK,oBAAoB,CAAC;AAAA,IAC7C,OAAO;AAEL,iBAAW,QAAQ,WAAW,MAAM;AAAA,QAClC,CAAC,SAAS,CAAC,KAAK,GAAG,WAAW,oBAAoB;AAAA,MACpD;AAAA,IACF;AAEA,UAAM,SAGA,CAAC;AAEP,UAAM,UAAU,kBAAI,mBAAmB,KAAK,SAAS,UAAU;AAC/D,eAAW,MAAM,QAAQ,UAAU;AACjC,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,SAAS;AAAA,QACT,cAAU,yBAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AAEA,eAAW,CAAC,YAAY,EAAE,KAAK,QAAQ,UAAU;AAC/C,YAAM,WAAW,WAAW,QAAQ,EAAE;AACtC,UAAI,CAAC,UAAU;AACb,cAAM,IAAI,MAAM,aAAa,EAAE,YAAY;AAAA,MAC7C;AACA,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,MAAM,wBAAwB,QAAQ;AAAA,QACtC,kBAAkB,cAAc;AAAA,QAChC,cAAU,yBAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,QAAwC;AACxD,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,KAAK,KAAK,uBAAuB,MAAM;AAC7C,SAAK,UAAU,EAAE;AAEjB,QAAI,CAAC,GAAG,QAAQ,OAAO;AACrB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAIA,UAAM,oBAAoB,IAAI,IAAI,GAAG,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC;AAC3E,UAAM,gBAAgB,OAAO;AAAA,MAC3B,OAAO,QAAQ,MAAM,EAAE;AAAA,QACrB,CAAC,CAAC,MAAM,IAAI,MAAM,kBAAI,eAAe,IAAI,KAAK,kBAAkB,IAAI,IAAI;AAAA,MAC1E;AAAA,IACF;AAEA,SAAK,SAAS;AAEd,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB,QAAuD;AACpF,UAAM,WAA6B,CAAC;AAEpC,eAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,UAAI,CAAC,kBAAI,eAAe,IAAI,GAAG;AAC7B,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAEA,YAAM,EAAE,YAAY,gBAAgB,YAAY,IAAI;AACpD,UAAI;AACF,cAAM,aAAa,kBAAI;AAAA,UACrB;AAAA,QACF;AAEA,iBAAS,KAAK;AAAA,UACZ;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR,CAAC;AAAA,MACH,SAAS,GAAG;AACV,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,OAAO;AAAA,MACT;AAAA,MACA,cAAU,yBAAU,eAAe;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,eAAsC;AAC7D,UAAM,cAAU,yBAAU,sBAAsB;AAChD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,QACP,cAAc;AAAA,MAChB;AAAA,MACA,UAAU;AAAA,IACZ,CAAiC;AACjC,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,cAAc,EAAE,WAAW,GAA0C;AACnE,UAAM,UAAmD,CAAC;AAE1D,SAAK,iBAAiB,SAAS,aAAa;AAC5C,YAAQ,cAAc,gBAAgB,UAAU;AAIhD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,MACT,cAAU,yBAAU,iBAAiB;AAAA,IACvC,CAAC;AAAA,EACH;AAAA,EAEA,UAAU,OAAyB;AACjC,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAM,GAAG;AAClD,aAAK,UAAU;AAAA,UACb,MAAM;AAAA,UACN,OAAO,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,QACtD,CAA0C;AAE1C,aAAK,oBAAqB,GAAG,oBAAoB,GAAG,aAAc;AAAA,MACpE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAA6B;AACjC,QAAI,KAAK,mBAAmB,KAAK;AAE/B,WAAK,UAAU;AAAA,QACb,MAAM;AAAA,MACR,CAA0C;AAC1C,WAAK,mBAAmB;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAyC;AACzC,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,UAAM,SAAS,KAAK,eAAe,EAAE,cAAc,eAAe,KAAK,CAAC;AACxE,SAAK,0BAA0B;AAC/B,WAAO,OAAO,QAAQ;AAAA,EACxB;AAAA,EAEA,MAAM,YAA2B;AAC/B,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAkC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,UAAoE;AACjF,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,eAAe;AAAA,MACf,SAAS,SAAS;AAAA,MAClB,cAAc,SAAS;AAAA,IACzB,CAA4C;AAAA,EAC9C;AAAA;AAAA;AAAA,EAIA,eACE,OACA,YAAoB,IACK;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,YAAM,gBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,cAAc;AAAA,IACjD;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,YAAM,iBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,eAAe;AAAA,IAClD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,eAAmC;AAC/C,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,IAChB;AAEA,QAAI,KAAK,iBAAiB,SAAS,SAAS;AAI1C,UAAI,KAAK,iBAAiB,SAAS,YAAY;AAC7C,gBAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,UAAU;AAAA,MAC7E,WAAW,KAAK,iBAAiB,SAAS,QAAQ;AAChD,gBAAQ,SAAS,IAAI,KAAK,iBAAiB,SAAS;AAAA,MACtD,OAAO;AACL,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAAA,IACF,OAAO;AACL,cAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,MAAM;AACvE,cAAQ,aAAa,IAAI;AAAA,IAC3B;AAEA,UAAM,MAAM,eAAe;AAAA,MACzB,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,OAAO,KAAK,iBAAiB,SAAS;AAAA,MACtC,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,YAAY,KAAK,iBAAiB,SAAS;AAAA,MAC3C,iBAAiB,KAAK,iBAAiB,SAAS;AAAA,IAClD,CAAC;AAED,SAAK,QAAQ,MAAM,wCAAwC,GAAG,EAAE;AAEhE,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,KAAK,IAAI,oBAAU,KAAK,EAAE,QAAQ,CAAC;AACzC,UAAI,UAAU;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,WAAG,MAAM;AACT,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,iBAAiB,SAAS,YAAY,SAAS;AAEvD,SAAG,KAAK,QAAQ,MAAM;AACpB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,gBAAQ,EAAE;AAAA,MACZ,CAAC;AAED,SAAG,KAAK,SAAS,MAAM;AACrB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,eAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YAA2B;AAC/B,QAAI,eAAe;AACnB,QAAI,aAAa;AACjB,QAAI,SAA2B;AAC/B,UAAM,aAAa,KAAK,iBAAiB,SAAS,YAAY;AAE9D,UAAM,YAAY,YAAY;AAC5B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,oBAAoB,KAAK,iBAAiB,SAAS;AAAA,QACrD;AAAA,QACA;AAAA,MACF;AAEA,YAAM,SAAkC,CAAC;AAGzC,aAAO,KAAK,KAAK,yBAAyB,CAAC;AAG3C,UAAI,OAAO,KAAK,KAAK,MAAM,EAAE,SAAS,GAAG;AACvC,eAAO,KAAK,KAAK,uBAAuB,KAAK,MAAM,CAAC;AAAA,MACtD;AAGA,YAAM,UAAU,KAAK,QAAQ,KAAK;AAAA,QAChC,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,MACvB,CAAC;AAED,YAAM,aAAa,KAAK;AACxB,WAAK,gBAAgB,IAAI,kBAAI,kBAAkB;AAC/C,aAAO,KAAK,GAAG,KAAK,0BAA0B,OAAO,CAAC;AAEtD,UAAI;AACF,mBAAW,MAAM,QAAQ;AACvB,eAAK,KAAK,8BAA8B,EAAE;AAC1C,iBAAQ,KAAK,KAAK,UAAU,EAAE,CAAC;AAAA,QACjC;AAAA,MACF,SAAS,OAAO;AACd,aAAK,gBAAgB;AACrB,cAAM,IAAI,iCAAmB;AAAA,UAC3B,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ,MAAM,oCAAoC;AAEvD,WAAK,KAAK,uBAAuB,CAAC,CAAwC;AAAA,IAC5E;AAEA,mBAAe;AACf,WAAO,CAAC,KAAK,SAAS;AACpB,WAAK,QAAQ,MAAM,sDAAsD;AACzE,eAAS,MAAM,KAAK,aAAa;AAEjC,UAAI;AACF,YAAI,cAAc;AAChB,gBAAM,UAAU;AAChB,uBAAa;AAAA,QACf;AACA,cAAM,KAAK,MAAM,MAAM;AAAA,MACzB,SAAS,OAAO;AACd,YAAI,KAAC,0BAAW,KAAK,GAAG;AACtB,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,KAAK,CAAC,MAAM,WAAW;AACxC,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,YAAY;AAC7B,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM,IAAI,iCAAmB;AAAA,YAC3B,SAAS,+CAA+C,UAAU;AAAA,YAClE,SAAS;AAAA,cACP,MAAM;AAAA,cACN,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAEA,aAAK,UAAU,EAAE,OAAuB,aAAa,KAAK,CAAC;AAC3D,cAAM,gBACJ,eAAe,IACX,kCACA,KAAK,iBAAiB,SAAS,YAAY;AACjD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS;AAAA,YACT;AAAA,YACA;AAAA,UACF;AAAA,UACA,sDAAsD,gBAAgB,GAAI;AAAA,QAC5E;AAEA,kBAAM,oBAAM,aAAa;AACzB;AAAA,MACF;AAEA,qBAAe;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAc,MAAM,QAAkC;AACpD,UAAM,gBAAgB,OAAO,WAAuC;AAClE,aAAO,CAAC,KAAK,WAAW,OAAO,eAAe,oBAAU,QAAQ,CAAC,OAAO,SAAS;AAC/E,YAAI;AACF,gBAAM,QAAQ,MAAM,KAAK,eAAe,IAAI;AAC5C,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AAEA,cAAI,MAAM,SAAS,6BAA6B;AAC9C,iBAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AAAA,UAChF;AAEA,eAAK,KAAK,8BAA8B,KAAK;AAC7C,iBAAO,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,QACnC,SAAS,OAAO;AACd;AAAA,QACF;AAAA,MACF;AAEA,aAAO,MAAM;AAAA,IACf;AAEA,UAAM,gBAAgB,IAAI,qBAAqB;AAE/C,WAAO,UAAU,CAAC,UAAU;AAC1B,oBAAc,QAAQ,IAAI,iCAAmB,EAAE,SAAS,MAAM,QAAQ,CAAC,CAAC;AAAA,IAC1E;AACA,WAAO,UAAU,MAAM;AACrB,oBAAc,QAAQ;AAAA,IACxB;AAEA,WAAO,YAAY,CAAC,YAA0B;AAC5C,YAAM,QAA+B,KAAK,MAAM,QAAQ,IAAc;AAEtE,WAAK,KAAK,gCAAgC,KAAK;AAC/C,WAAK,QAAQ,MAAM,eAAe,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AAE9E,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,sBAAsB,KAAK;AAChC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,uDAAuD,KAAK;AACjE;AAAA,QACF,KAAK;AACH,eAAK,oDAAoD,KAAK;AAC9D;AAAA,QACF,KAAK;AACH,eAAK,+BAA+B,KAAK;AACzC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,mCAAmC,KAAK;AAC7C;AAAA,QACF,KAAK;AACH,eAAK,yBAAyB,KAAK;AACnC;AAAA,QACF,KAAK;AACH,eAAK,kCAAkC,KAAK;AAC5C;AAAA,QACF,KAAK;AACH,eAAK,wBAAwB,KAAK;AAClC;AAAA,QACF,KAAK;AACH,eAAK,6BAA6B,KAAK;AACvC;AAAA,QACF,KAAK;AACH,eAAK,mBAAmB,KAAK;AAC7B;AAAA,QACF,KAAK;AACH,eAAK,YAAY,KAAK;AACtB;AAAA,QACF;AACE,eAAK,QAAQ,MAAM,oBAAoB,MAAM,IAAI,EAAE;AACnD;AAAA,MACJ;AAAA,IACF;AAEA,UAAM,WAAW,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,cAAc,MAAM,CAAC;AAEhE,UAAM,SAAS,mBAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AACvC,YAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,eAAO,iBAAiB,SAAS,MAAM;AACrC,kBAAQ;AAAA,QACV,CAAC;AAAA,MACH,CAAC;AAED,aAAO,QAAQ,KAAK,CAAC,cAAc,OAAO,YAAY,CAAC;AAAA,IACzD,CAAC;AAED,UAAM,oBAAoB,mBAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AACxD,gBAAM,oBAAM,KAAK,iBAAiB,SAAS,oBAAoB,EAAE,OAAO,CAAC;AACzE,aAAO,IAAI,iCAAmB;AAAA,QAC5B,SAAS;AAAA,MACX,CAAC;AAAA,IACH,CAAC;AAED,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,OAAO,QAAQ,SAAS,QAAQ,kBAAkB,MAAM,CAAC;AAE5F,UAAI,kBAAkB,QAAQ,KAAK,mBAAmB;AACpD,cAAM,KAAK,kBAAkB,SAAS;AAAA,MACxC;AAEA,UAAI,kBAAkB,OAAO;AAC3B,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,gBAAM,6BAAc,CAAC,QAAQ,UAAU,iBAAiB,GAAG,GAAI;AAC/D,aAAO,MAAM;AAAA,IACf;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,MAAM;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B,KAAK,iBAAiB,SAAS,4BAA4B;AAAA,IACvF,CAAgC;AAAA,EAClC;AAAA,EAEQ,sBAAsB,OAA6C;AACzE,QAAI,CAAC,MAAM,SAAS,IAAI;AACtB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,qBAAO,oBAA2C;AAAA,MAClE,iBAAiB,qBAAO,oBAAsC;AAAA,MAC9D,UAAU,oBAAI,IAAI;AAAA,MAClB,UAAU,IAAI,qBAAO;AAAA,MACrB,mBAAmB,KAAK,IAAI;AAAA,IAC9B;AAEA,QAAI,CAAC,MAAM,SAAS,YAAY,CAAC,MAAM,SAAS,SAAS,gBAAiB;AAE1E,UAAM,SAAS,KAAK,uBAAuB,MAAM,SAAS,SAAS,eAAe;AAClF,QAAI,QAAQ;AACV,aAAO,KAAK,uBAAuB,MAAM,SAAS,SAAS,eAAe;AAG1E,WAAK,uBAAuB,MAAM,SAAS,EAAE,IAAI;AAAA,IACnD;AAMA,SAAK,KAAK,sBAAsB;AAAA,MAC9B,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB,CAA2B;AAAA,EAC7B;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,QAAI,CAAC,MAAM,KAAK,MAAM;AACpB,YAAM,IAAI,MAAM,sBAAsB;AAAA,IACxC;AAEA,QAAI,CAAC,MAAM,aAAa;AACtB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,UAAM,WAAW,MAAM,KAAK;AAC5B,UAAM,aAAa,MAAM;AAEzB,QAAI,aAAa,WAAW;AAE1B,WAAK,oBAAoB,UAAU;AACnC,WAAK,0BAA0B;AAC/B;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,KAAK,IAAI;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,kBAAkB,wBAAwB,MAAM,IAAI,CAAC;AAAA,IACvF,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,KAAK,GAAG,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,OAAO;AAAA,IACzC,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,QAAQ,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,OAAO;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,OAAO;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,uDACN,OACM;AACN,UAAM,aAAa,KAAK,cAAc,IAAI,MAAM,OAAO;AACvD,QAAI,CAAC,YAAY;AACf;AAAA,IACF;AAEA,UAAM,OAAO,WAAW;AACxB,QAAI,gBAAgB,kBAAI,aAAa;AACnC,WAAK,QAAQ,KAAK,MAAM,UAAU;AAAA,IACpC,OAAO;AACL,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,SAAK,KAAK,uCAAuC;AAAA,MAC/C,QAAQ,MAAM;AAAA,MACd,YAAY,MAAM;AAAA,MAClB,SAAS;AAAA,IACX,CAAoC;AAAA,EACtC;AAAA,EAEQ,oDACN,OACM;AACN,SAAK,QAAQ;AAAA,MACX,EAAE,OAAO,MAAM,MAAM;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,+BAA+B,OAAsD;AAC3F,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AACrB,UAAM,WAAW,MAAM,KAAK;AAC5B,UAAM,aAAa,MAAM;AAEzB,QAAI,aAAa,SAAS;AACxB,WAAK,oBAAoB,UAAU;AACnC,UAAI,KAAK,0BAA0B,GAAG;AACpC,aAAK,QAAQ;AAAA,UACX,EAAE,SAAS,KAAK,wBAAwB;AAAA,UACxC;AAAA,QACF;AACA,aAAK,0BAA0B;AAAA,MACjC;AAEA,YAAM,iBAAoC;AAAA,QACxC,WAAW;AAAA,QACX,aAAa,qBAAO,oBAA4B;AAAA,QAChD,cAAc,qBAAO,oBAAgC;AAAA,QACrD,iBAAiB;AAAA,MACnB;AAEA,WAAK,kBAAkB,eAAe,MAAM;AAAA,QAC1C,WAAW;AAAA,QACX,YAAY,eAAe,YAAY,OAAO;AAAA,QAC9C,aAAa,eAAe,aAAa,OAAO;AAAA,MAClD,CAAC;AAED,WAAK,kBAAkB,SAAS,IAAI,QAAQ,cAAc;AAC1D,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AACvD;AAAA,IACF,OAAO;AACL,WAAK,UAAU;AACf,UAAI,KAAK,4BAA4B,GAAG;AACtC,aAAK,QAAQ,KAAK,EAAE,WAAW,GAAG,sDAAsD;AAAA,MAC1F;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,MAAM,KAAK,SAAS,QAAQ;AAC9B;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EAGF;AAAA,EAEQ,mCACN,OACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AACrB,UAAM,QAAQ,MAAM;AAIpB,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C,OAAO;AACL,qBAAe,YAAY,MAAM,KAAK;AACtC,qBAAe,mBAAmB;AAAA,IACpC;AAAA,EACF;AAAA,EAEQ,yBAAyB,OAAgD;AAC/E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM,OAAO;AACxE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,UAAM,eAAe,KAAK,MAAM,KAAK;AACrC,UAAM,MAAM,aAAa;AACzB,UAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,aAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,YAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,IACtC;AAEA,mBAAe,aAAa;AAAA,MAC1B,IAAI;AAAA,QACF,IAAI,WAAW,MAAM,MAAM;AAAA,QAC3B,UAAU;AAAA,QACV,UAAU;AAAA,QACV,MAAM,SAAS;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kCACN,QACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,wBAAwB,QAAgD;AAC9E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,6BAA6B,OAAoD;AACvF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM,KAAK;AAC1B,UAAM,WAAW,MAAM,KAAK;AAE5B,QAAI,aAAa,iBAAiB;AAChC,YAAM,OAAO,MAAM;AACnB,UAAI,CAAC,KAAK,WAAW,CAAC,KAAK,QAAQ,CAAC,KAAK,WAAW;AAClD,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,WAAK,kBAAkB,gBAAgB,MAAM;AAAA,QAC3C,QAAQ,KAAK;AAAA,QACb,MAAM,KAAK;AAAA,QACX,MAAM,KAAK;AAAA,MACb,CAAqB;AAAA,IACvB,WAAW,aAAa,WAAW;AACjC,YAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,UAAI,CAAC,gBAAgB;AACnB;AAAA,MACF;AAEA,qBAAe,YAAY,MAAM;AACjC,qBAAe,aAAa,MAAM;AAAA,IACpC;AAAA,EACF;AAAA,EAEQ,mBAAmB,QAA2C;AA1uCxE;AA2uCI,QAAI,CAAC,KAAK,mBAAmB;AAG3B;AAAA,IACF;AAEA,UAAM,mBAAmB,KAAK,kBAAkB;AAChD,UAAM,sBAAsB,KAAK,kBAAkB;AAEnD,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,cAAc,KAAK,kBAAkB,SAAS;AAAA,MAChD;AAAA,MACA;AAAA,IACF;AAEA,eAAW,cAAc,KAAK,kBAAkB,SAAS,OAAO,GAAG;AACjE,iBAAW,YAAY,MAAM;AAC7B,iBAAW,aAAa,MAAM;AAAA,IAChC;AAEA,SAAK,kBAAkB,gBAAgB,MAAM;AAC7C,SAAK,kBAAkB,eAAe,MAAM;AAE5C,eAAW,UAAU,KAAK,kBAAkB,SAAS,KAAK,GAAG;AAC3D,YAAM,aAAa,KAAK,cAAc,IAAI,MAAM;AAChD,UAAI,cAAc,WAAW,gBAAgB,kBAAI,aAAa;AAC5D,mBAAW,KAAK,QAAQ,KAAK,KAAK,kBAAkB,SAAS,IAAI,MAAM,EAAG,eAAe;AAAA,MAC3F;AAAA,IACF;AAEA,SAAK,kBAAkB,SAAS,QAAQ;AACxC,SAAK,oBAAoB;AAGzB,UAAM,QAAQ,OAAO,SAAS;AAC9B,UAAM,OAAO,sBAAsB,sBAAsB,mBAAmB;AAC5E,UAAM,YAAY,KAAK,IAAI,IAAI,oBAAoB;AAEnD,UAAM,kBAAgD;AAAA,MACpD,MAAM;AAAA,MACN,WAAW,mBAAmB;AAAA;AAAA,MAC9B,WAAW,OAAO,SAAS,MAAM;AAAA,MACjC;AAAA,MACA;AAAA,MACA,WAAW,OAAO,SAAS,WAAW;AAAA,MACtC,OAAO;AAAA,MACP,cAAa,+BAAO,iBAAgB;AAAA,MACpC,eAAc,+BAAO,kBAAiB;AAAA,MACtC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,iBAAiB,WAAW,MAAK,+BAAO,kBAAiB,KAAK,WAAW;AAAA,MACzE,mBAAmB;AAAA,QACjB,eAAa,oCAAO,wBAAP,mBAA4B,iBAAgB;AAAA,QACzD,cAAY,oCAAO,wBAAP,mBAA4B,gBAAe;AAAA,QACvD,aAAa;AAAA;AAAA,QACb,gBAAc,oCAAO,wBAAP,mBAA4B,kBAAiB;AAAA,QAC3D,uBAAqB,oCAAO,wBAAP,mBAA4B,yBAC7C;AAAA,UACE,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,UAChF,cAAY,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,gBAAe;AAAA,UAC9E,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,QAClF,IACA;AAAA,MACN;AAAA,MACA,oBAAoB;AAAA,QAClB,cAAY,oCAAO,yBAAP,mBAA6B,gBAAe;AAAA,QACxD,eAAa,oCAAO,yBAAP,mBAA6B,iBAAgB;AAAA,QAC1D,aAAa;AAAA,MACf;AAAA,IACF;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAEhD;AAAA,EAEQ,YAAY,OAAmC;AACrD,QAAI,MAAM,MAAM,QAAQ,WAAW,qBAAqB,GAAG;AACzD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,OAAO,MAAM,MAAM,GAAG,uCAAuC;AAClF,SAAK,UAAU;AAAA,MACb,OAAO,IAAI,uBAAS,MAAM,MAAM,SAAS;AAAA,QACvC,MAAM,MAAM;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAAA,MACD,aAAa;AAAA,IACf,CAAC;AAAA,EAGH;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAAiD;AAEtF,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAA2B;AAAA,EAC7B;AAAA,EAEA,CAAS,cAAc,OAA0C;AAC/D,UAAM;AAAA,EACR;AAAA,EAEQ,eAAe;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIyB;AACvB,UAAM,SAAS,aAAa,IAAI,qBAAqB,EAAE,aAAa,CAAC;AACrE,QAAI,aAAa,cAAc;AAC7B,aAAO,eAAe;AAAA,IACxB;AAEA,UAAM,cAAU,yBAAU,kBAAkB;AAC5C,QAAI,eAAe;AACjB,WAAK,uBAAuB,OAAO,IAAI;AAAA,IACzC;AAEA,UAAM,WAAsD,CAAC;AAC7D,QAAI,aAAc,UAAS,eAAe;AAC1C,QAAI,cAAe,UAAS,WAAW,EAAE,iBAAiB,QAAQ;AAElE,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,UAAU;AAAA,MACV,UAAU,OAAO,KAAK,QAAQ,EAAE,SAAS,IAAI,WAAW;AAAA,IAC1D,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,YAA0B;AACpD,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,gBAA4C;AAAA,MAChD,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,IACjB;AAEA,UAAM,SAAS,KAAK,uBAAuB,UAAU;AACrD,QAAI,QAAQ;AACV,aAAO,KAAK,uBAAuB,UAAU;AAC7C,oBAAc,gBAAgB;AAC9B,UAAI,OAAO,QAAQ,MAAM;AACvB,aAAK,QAAQ,KAAK,EAAE,WAAW,GAAG,iCAAiC;AAAA,MACrE,OAAO;AACL,eAAO,QAAQ,QAAQ,aAAa;AAAA,MACtC;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,WAAW,GAAG,mCAAmC;AACtE,SAAK,KAAK,sBAAsB,aAAa;AAAA,EAC/C;AACF;AAEA,SAAS,wBAAwB,MAA4C;AAC3E,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,MAAM,KAAK;AAAA,QACX,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,QAAQ,KAAK;AAAA,MACf;AAAA,IACF,KAAK;AACH,YAAM,OAAO,KAAK,SAAS,cAAc,WAAW,KAAK;AACzD,YAAM,cAAmC,CAAC;AAC1C,iBAAW,KAAK,KAAK,SAAS;AAC5B,YAAI,OAAO,MAAM,UAAU;AACzB,sBAAY,KAAK;AAAA,YACf,MAAM,SAAS,cAAc,SAAS;AAAA,YACtC,MAAM;AAAA,UACR,CAA+B;AAAA,QACjC,WAAW,EAAE,SAAS,iBAAiB;AAErC;AAAA,QACF,WAAW,EAAE,SAAS,iBAAiB;AACrC,cAAI,SAAS,QAAQ;AACnB,kBAAM,eAAe,OAAO,SAAK,oCAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,QAAQ;AACpF,wBAAY,KAAK;AAAA,cACf,MAAM;AAAA,cACN,OAAO;AAAA,YACT,CAAgC;AAAA,UAClC;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN;AAAA,QACA,SAAS;AAAA,MACX;AAAA,EACJ;AACF;AAEA,SAAS,wBAAwB,MAA4C;AAC3E,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,oBAAoB;AAAA,EACtC;AAEA,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO,kBAAI,aAAa,OAAO;AAAA,QAC7B,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,MAAM,KAAK;AAAA,QACX,MAAM,KAAK;AAAA,MACb,CAAC;AAAA,IACH,KAAK;AACH,aAAO,kBAAI,mBAAmB,OAAO;AAAA,QACnC,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,QAAQ,KAAK;AAAA,QACb,SAAS;AAAA,MACX,CAAC;AAAA,IACH,KAAK;AACH,YAAM,UAA6B,CAAC;AAEpC,YAAM,WAAW,MAAM,QAAQ,KAAK,OAAO,IAAI,KAAK,UAAU,CAAC,KAAK,OAAO;AAC3E,iBAAW,KAAK,UAAU;AACxB,YAAI,EAAE,SAAS,UAAU,EAAE,SAAS,cAAc;AAChD,kBAAQ,KAAK,EAAE,IAAI;AAAA,QACrB;AAAA,MACF;AACA,aAAO,kBAAI,YAAY,OAAO;AAAA,QAC5B,IAAI,KAAK;AAAA,QACT,MAAM,KAAK;AAAA,QACX;AAAA,MACF,CAAC;AAAA,EACL;AACF;AAEA,SAAS,oBAAoB,kBAA0B,GAAoB;AACzE,QAAM,YAAY,OAAO,MAAM,kBAAkB,WAAW;AAC5D,SAAO,kBAAI,YAAY,OAAO;AAAA,IAC5B,QAAI,yBAAU,oBAAoB;AAAA,IAClC,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,QACE,MAAM;AAAA,QACN,OAAO;AAAA,UACL,IAAI;AAAA,YACF,IAAI,WAAW,UAAU,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,YACA,UAAU,SAAS;AAAA,UACrB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,gBAAgB,YAAmD;AAC1E,MAAI,OAAO,eAAe,UAAU;AAClC,WAAO;AAAA,EACT;AAEA,OAAI,yCAAY,UAAS,YAAY;AACnC,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,SAAO;AACT;","names":[]}

package/dist/realtime/realtime_model.d.cts

@@ -1,190 +1,156 @@
-import { AsyncIterableQueue, Future, llm, multimodal } from '@livekit/agents';
+import { type APIConnectOptions, llm } from '@livekit/agents';
 import { AudioFrame } from '@livekit/rtc-node';
 import * as api_proto from './api_proto.js';
-interface ModelOptions {
-    modalities: ['text', 'audio'] | ['text'];
-    instructions: string;
+interface RealtimeOptions {
+    model: api_proto.Model;
     voice: api_proto.Voice;
-    inputAudioFormat: api_proto.AudioFormat;
-    outputAudioFormat: api_proto.AudioFormat;
-    inputAudioTranscription: api_proto.InputAudioTranscription | null;
-    turnDetection: api_proto.TurnDetectionType | null;
     temperature: number;
-    maxResponseOutputTokens: number;
-    model: api_proto.Model;
+    toolChoice?: llm.ToolChoice;
+    inputAudioTranscription?: api_proto.InputAudioTranscription | null;
+    turnDetection?: api_proto.TurnDetectionType | null;
+    maxResponseOutputTokens?: number | 'inf';
+    speed?: number;
     apiKey?: string;
     baseURL: string;
     isAzure: boolean;
+    azureDeployment?: string;
     entraToken?: string;
     apiVersion?: string;
+    maxSessionDuration: number;
+    connOptions: APIConnectOptions;
 }
-export interface RealtimeResponse {
-    id: string;
-    status: api_proto.ResponseStatus;
-    statusDetails: api_proto.ResponseStatusDetails | null;
-    usage: api_proto.ModelUsage | null;
-    output: RealtimeOutput[];
-    doneFut: Future;
-    createdTimestamp: number;
-    firstTokenTimestamp?: number;
-}
-export interface RealtimeOutput {
-    responseId: string;
-    itemId: string;
-    outputIndex: number;
-    role: api_proto.Role;
-    type: 'message' | 'function_call';
-    content: RealtimeContent[];
-    doneFut: Future;
-}
-export interface RealtimeContent {
-    responseId: string;
-    itemId: string;
-    outputIndex: number;
-    contentIndex: number;
-    text: string;
-    audio: AudioFrame[];
-    textStream: AsyncIterableQueue<string>;
-    audioStream: AsyncIterableQueue<AudioFrame>;
-    toolCalls: RealtimeToolCall[];
-    contentType: api_proto.Modality;
-}
-export interface RealtimeToolCall {
-    name: string;
-    arguments: string;
-    toolCallID: string;
-}
-export interface InputSpeechTranscriptionCompleted {
-    itemId: string;
-    transcript: string;
-}
-export interface InputSpeechTranscriptionFailed {
-    itemId: string;
-    message: string;
-}
-export interface InputSpeechStarted {
-    itemId: string;
-}
-export interface InputSpeechCommitted {
-    itemId: string;
-}
-declare class InputAudioBuffer {
-    #private;
-    constructor(session: RealtimeSession);
-    append(frame: AudioFrame): void;
-    clear(): void;
-    commit(): void;
-}
-declare class ConversationItem {
-    #private;
-    constructor(session: RealtimeSession);
-    truncate(itemId: string, contentIndex: number, audioEnd: number): void;
-    delete(itemId: string): void;
-    create(message: llm.ChatMessage, previousItemId?: string): void;
-}
-declare class Conversation {
-    #private;
-    constructor(session: RealtimeSession);
-    get item(): ConversationItem;
-}
-declare class Response {
-    #private;
-    constructor(session: RealtimeSession);
-    create(): void;
-    cancel(): void;
-}
-export declare class RealtimeModel extends multimodal.RealtimeModel {
-    #private;
+export declare class RealtimeModel extends llm.RealtimeModel {
     sampleRate: number;
     numChannels: number;
     inFrameSize: number;
     outFrameSize: number;
-    static withAzure({ baseURL, azureDeployment, apiVersion, apiKey, entraToken, instructions, modalities, voice, inputAudioFormat, outputAudioFormat, inputAudioTranscription, turnDetection, temperature, maxResponseOutputTokens, }: {
-        baseURL: string;
+    _options: RealtimeOptions;
+    constructor(options?: {
+        model?: string;
+        voice?: string;
+        temperature?: number;
+        toolChoice?: llm.ToolChoice;
+        baseURL?: string;
+        inputAudioTranscription?: api_proto.InputAudioTranscription | null;
+        turnDetection?: api_proto.TurnDetectionType | null;
+        speed?: number;
+        azureDeployment?: string;
+        apiKey?: string;
+        entraToken?: string;
+        apiVersion?: string;
+        maxSessionDuration?: number;
+        connOptions?: APIConnectOptions;
+    });
+    /**
+     * Create a RealtimeModel instance configured for Azure OpenAI Service.
+     *
+     * @param azureDeployment - The name of your Azure OpenAI deployment.
+     * @param azureEndpoint - The endpoint URL for your Azure OpenAI resource. If undefined, will attempt to read from the environment variable AZURE_OPENAI_ENDPOINT.
+     * @param apiVersion - API version to use with Azure OpenAI Service. If undefined, will attempt to read from the environment variable OPENAI_API_VERSION.
+     * @param apiKey - Azure OpenAI API key. If undefined, will attempt to read from the environment variable AZURE_OPENAI_API_KEY.
+     * @param entraToken - Azure Entra authentication token. Required if not using API key authentication.
+     * @param baseURL - Base URL for the API endpoint. If undefined, constructed from the azure_endpoint.
+     * @param voice - Voice setting for audio outputs. Defaults to "alloy".
+     * @param inputAudioTranscription - Options for transcribing input audio. Defaults to @see DEFAULT_INPUT_AUDIO_TRANSCRIPTION.
+     * @param turnDetection - Options for server-based voice activity detection (VAD). Defaults to @see DEFAULT_SERVER_VAD_OPTIONS.
+     * @param temperature - Sampling temperature for response generation. Defaults to @see DEFAULT_TEMPERATURE.
+     * @param speed - Speed of the audio output. Defaults to 1.0.
+     * @param maxResponseOutputTokens - Maximum number of tokens in the response. Defaults to @see DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS.
+     * @param maxSessionDuration - Maximum duration of the session in milliseconds. Defaults to @see DEFAULT_MAX_SESSION_DURATION.
+     *
+     * @returns A RealtimeModel instance configured for Azure OpenAI Service.
+     *
+     * @throws Error if required Azure parameters are missing or invalid.
+     */
+    static withAzure({ azureDeployment, azureEndpoint, apiVersion, apiKey, entraToken, baseURL, voice, inputAudioTranscription, turnDetection, temperature, speed, }: {
         azureDeployment: string;
+        azureEndpoint?: string;
         apiVersion?: string;
         apiKey?: string;
         entraToken?: string;
-        instructions?: string;
-        modalities?: ['text', 'audio'] | ['text'];
-        voice?: api_proto.Voice;
-        inputAudioFormat?: api_proto.AudioFormat;
-        outputAudioFormat?: api_proto.AudioFormat;
+        baseURL?: string;
+        voice?: string;
         inputAudioTranscription?: api_proto.InputAudioTranscription;
         turnDetection?: api_proto.TurnDetectionType;
         temperature?: number;
-        maxResponseOutputTokens?: number;
+        speed?: number;
     }): RealtimeModel;
-    constructor({ modalities, instructions, voice, inputAudioFormat, outputAudioFormat, inputAudioTranscription, turnDetection, temperature, maxResponseOutputTokens, model, apiKey, baseURL, isAzure, apiVersion, entraToken, }: {
-        modalities?: ['text', 'audio'] | ['text'];
-        instructions?: string;
-        voice?: api_proto.Voice;
-        inputAudioFormat?: api_proto.AudioFormat;
-        outputAudioFormat?: api_proto.AudioFormat;
-        inputAudioTranscription?: api_proto.InputAudioTranscription;
-        turnDetection?: api_proto.TurnDetectionType;
-        temperature?: number;
-        maxResponseOutputTokens?: number;
-        model?: api_proto.Model;
-        apiKey?: string;
-        baseURL?: string;
-        isAzure?: boolean;
-        apiVersion?: string;
-        entraToken?: string;
-    });
-    get sessions(): RealtimeSession[];
-    session({ fncCtx, chatCtx, modalities, instructions, voice, inputAudioFormat, outputAudioFormat, inputAudioTranscription, turnDetection, temperature, maxResponseOutputTokens, }: {
-        fncCtx?: llm.FunctionContext;
-        chatCtx?: llm.ChatContext;
-        modalities?: ['text', 'audio'] | ['text'];
-        instructions?: string;
-        voice?: api_proto.Voice;
-        inputAudioFormat?: api_proto.AudioFormat;
-        outputAudioFormat?: api_proto.AudioFormat;
-        inputAudioTranscription?: api_proto.InputAudioTranscription | null;
-        turnDetection?: api_proto.TurnDetectionType | null;
-        temperature?: number;
-        maxResponseOutputTokens?: number;
-    }): RealtimeSession;
+    session(): RealtimeSession;
     close(): Promise<void>;
 }
-export declare class RealtimeSession extends multimodal.RealtimeSession {
+/**
+ * A session for the OpenAI Realtime API.
+ *
+ * This class is used to interact with the OpenAI Realtime API.
+ * It is responsible for sending events to the OpenAI Realtime API and receiving events from it.
+ *
+ * It exposes two more events:
+ * - openai_server_event_received: expose the raw server events from the OpenAI Realtime API
+ * - openai_client_event_queued: expose the raw client events sent to the OpenAI Realtime API
+ */
+export declare class RealtimeSession extends llm.RealtimeSession {
     #private;
-    constructor(opts: ModelOptions, { fncCtx, chatCtx }: {
-        fncCtx?: llm.FunctionContext;
-        chatCtx?: llm.ChatContext;
-    });
-    get chatCtx(): llm.ChatContext | undefined;
-    get fncCtx(): llm.FunctionContext | undefined;
-    set fncCtx(ctx: llm.FunctionContext | undefined);
-    get conversation(): Conversation;
-    get inputAudioBuffer(): InputAudioBuffer;
-    get response(): Response;
-    get expiration(): number;
-    queueMsg(command: api_proto.ClientEvent): void;
-    sessionUpdate({ modalities, instructions, voice, inputAudioFormat, outputAudioFormat, inputAudioTranscription, turnDetection, temperature, maxResponseOutputTokens, toolChoice, selectedTools, }: {
-        modalities: ['text', 'audio'] | ['text'];
-        instructions?: string;
-        voice?: api_proto.Voice;
-        inputAudioFormat?: api_proto.AudioFormat;
-        outputAudioFormat?: api_proto.AudioFormat;
-        inputAudioTranscription?: api_proto.InputAudioTranscription | null;
-        turnDetection?: api_proto.TurnDetectionType | null;
-        temperature?: number;
-        maxResponseOutputTokens?: number;
-        toolChoice?: api_proto.ToolChoice;
-        selectedTools?: string[];
+    private _tools;
+    private remoteChatCtx;
+    private messageChannel;
+    private inputResampler?;
+    private instructions?;
+    private oaiRealtimeModel;
+    private currentGeneration?;
+    private responseCreatedFutures;
+    private textModeRecoveryRetries;
+    private itemCreateFutures;
+    private itemDeleteFutures;
+    private updateChatCtxLock;
+    private updateFuncCtxLock;
+    private bstream;
+    private pushedDurationMs;
+    constructor(realtimeModel: RealtimeModel);
+    sendEvent(command: api_proto.ClientEvent): void;
+    private createSessionUpdateEvent;
+    get chatCtx(): llm.ChatContext;
+    get tools(): llm.ToolContext;
+    updateChatCtx(_chatCtx: llm.ChatContext): Promise<void>;
+    private createChatCtxUpdateEvents;
+    updateTools(_tools: llm.ToolContext): Promise<void>;
+    private createToolsUpdateEvent;
+    updateInstructions(_instructions: string): Promise<void>;
+    updateOptions({ toolChoice }: {
+        toolChoice?: llm.ToolChoice;
     }): void;
-    /**
-     * Try to recover from a text response to audio mode.
-     *
-     * @remarks
-     * Sometimes the OpenAI Realtime API returns text instead of audio responses.
-     * This method tries to recover from this by requesting a new response after deleting the text
-     * response and creating an empty user audio message.
-     */
-    recoverFromTextResponse(itemId: string): void;
+    pushAudio(frame: AudioFrame): void;
+    commitAudio(): Promise<void>;
+    clearAudio(): Promise<void>;
+    generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent>;
+    interrupt(): Promise<void>;
+    truncate(_options: {
+        messageId: string;
+        audioEndMs: number;
+    }): Promise<void>;
+    private createWsConn;
+    private runWs;
     close(): Promise<void>;
+    private handleInputAudioBufferSpeechStarted;
+    private handleInputAudioBufferSpeechStopped;
+    private handleResponseCreated;
+    private handleResponseOutputItemAdded;
+    private handleConversationItemCreated;
+    private handleConversationItemDeleted;
+    private handleConversationItemInputAudioTranscriptionCompleted;
+    private handleConversationItemInputAudioTranscriptionFailed;
+    private handleResponseContentPartAdded;
+    private handleResponseContentPartDone;
+    private handleResponseAudioTranscriptDelta;
+    private handleResponseAudioDelta;
+    private handleResponseAudioTranscriptDone;
+    private handleResponseAudioDone;
+    private handleResponseOutputItemDone;
+    private handleResponseDone;
+    private handleError;
+    private emitError;
+    private resampleAudio;
+    private createResponse;
+    private emitGenerationEvent;
 }
 export {};
 //# sourceMappingURL=realtime_model.d.ts.map