@livekit/agents-plugin-openai 1.0.30 → 1.0.32

package/dist/realtime/realtime_model_beta.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/realtime/realtime_model_beta.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { metrics } from '@livekit/agents';\nimport {\n  type APIConnectOptions,\n  APIConnectionError,\n  APIError,\n  AudioByteStream,\n  DEFAULT_API_CONNECT_OPTIONS,\n  Future,\n  Queue,\n  Task,\n  cancelAndWait,\n  delay,\n  isAPIError,\n  llm,\n  log,\n  shortuuid,\n  stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport type { AudioResampler } from '@livekit/rtc-node';\nimport { AudioFrame, combineAudioFrames } from '@livekit/rtc-node';\nimport { type MessageEvent, WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\n// if LK_OPENAI_DEBUG convert it to a number, otherwise set it to 0\nconst lkOaiDebug = process.env.LK_OPENAI_DEBUG ? Number(process.env.LK_OPENAI_DEBUG) : 0;\n\nconst SAMPLE_RATE = 24000;\nconst NUM_CHANNELS = 1;\nconst BASE_URL = 'https://api.openai.com/v1';\n\nconst MOCK_AUDIO_ID_PREFIX = 'lk_mock_audio_item_';\n\ntype Modality = 'text' | 'audio';\n\ninterface RealtimeOptions {\n  model: api_proto.Model;\n  voice: api_proto.Voice;\n  temperature: number;\n  toolChoice?: llm.ToolChoice;\n  inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n  // TODO(shubhra): add inputAudioNoiseReduction\n  turnDetection?: api_proto.TurnDetectionType | null;\n  maxResponseOutputTokens?: number | 'inf';\n  speed?: number;\n  // TODO(shubhra): add openai tracing options\n  apiKey?: string;\n  baseURL: string;\n  isAzure: boolean;\n  azureDeployment?: string;\n  entraToken?: string;\n  apiVersion?: string;\n  maxSessionDuration: number;\n  // reset the connection after this many seconds if provided\n  connOptions: APIConnectOptions;\n  modalities: Modality[];\n}\n\ninterface MessageGeneration {\n  messageId: string;\n  textChannel: stream.StreamChannel<string>;\n  audioChannel: stream.StreamChannel<AudioFrame>;\n  audioTranscript: string;\n  modalities: Future<('text' | 'audio')[]>;\n}\n\ninterface ResponseGeneration {\n  messageChannel: stream.StreamChannel<llm.MessageGeneration>;\n  functionChannel: stream.StreamChannel<llm.FunctionCall>;\n  messages: Map<string, MessageGeneration>;\n\n  /** @internal */\n  _doneFut: Future;\n  /** @internal */\n  _createdTimestamp: number;\n  /** @internal */\n  _firstTokenTimestamp?: number;\n}\n\nclass CreateResponseHandle {\n  instructions?: string;\n  doneFut: Future<llm.GenerationCreatedEvent>;\n  // TODO(shubhra): add timeout\n  constructor({ instructions }: { instructions?: string }) {\n    this.instructions = instructions;\n    this.doneFut = new Future();\n  }\n}\n\n// default values got from a \"default\" session from their API\nconst DEFAULT_FIRST_RETRY_INTERVAL_MS = 100;\nconst DEFAULT_TEMPERATURE = 0.8;\nconst DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n  type: 'semantic_vad',\n  eagerness: 'medium',\n  create_response: true,\n  interrupt_response: true,\n};\nconst DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n  model: 'gpt-4o-mini-transcribe',\n};\nconst DEFAULT_TOOL_CHOICE: llm.ToolChoice = 'auto';\nconst DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS: number | 'inf' = 'inf';\n\nconst AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n  model: 'whisper-1',\n};\n\nconst AZURE_DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n  type: 'server_vad',\n  threshold: 0.5,\n  prefix_padding_ms: 300,\n  silence_duration_ms: 200,\n  create_response: true,\n};\n\nconst DEFAULT_MAX_SESSION_DURATION = 20 * 60 * 1000; // 20 minutes\n\nconst DEFAULT_REALTIME_MODEL_OPTIONS = {\n  model: 'gpt-realtime',\n  voice: 'marin',\n  temperature: DEFAULT_TEMPERATURE,\n  inputAudioTranscription: DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n  turnDetection: DEFAULT_TURN_DETECTION,\n  toolChoice: DEFAULT_TOOL_CHOICE,\n  maxResponseOutputTokens: DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS,\n  maxSessionDuration: DEFAULT_MAX_SESSION_DURATION,\n  connOptions: DEFAULT_API_CONNECT_OPTIONS,\n  modalities: ['text', 'audio'] as Modality[],\n};\nexport class RealtimeModel extends llm.RealtimeModel {\n  sampleRate = api_proto.SAMPLE_RATE;\n  numChannels = api_proto.NUM_CHANNELS;\n  inFrameSize = api_proto.IN_FRAME_SIZE;\n  outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n  /* @internal */\n  _options: RealtimeOptions;\n\n  get model(): string {\n    return this._options.model;\n  }\n\n  constructor(\n    options: {\n      model?: string;\n      voice?: string;\n      temperature?: number;\n      toolChoice?: llm.ToolChoice;\n      baseURL?: string;\n      modalities?: Modality[];\n      inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n      // TODO(shubhra): add inputAudioNoiseReduction\n      turnDetection?: api_proto.TurnDetectionType | null;\n      speed?: number;\n      // TODO(shubhra): add openai tracing options\n      azureDeployment?: string;\n      apiKey?: string;\n      entraToken?: string;\n      apiVersion?: string;\n      maxSessionDuration?: number;\n      connOptions?: APIConnectOptions;\n    } = {},\n  ) {\n    const modalities = (options.modalities ||\n      DEFAULT_REALTIME_MODEL_OPTIONS.modalities) as Modality[];\n\n    super({\n      messageTruncation: true,\n      turnDetection: options.turnDetection !== null,\n      userTranscription: options.inputAudioTranscription !== null,\n      autoToolReplyGeneration: false,\n      audioOutput: modalities.includes('audio'),\n    });\n\n    const isAzure = !!(options.apiVersion || options.entraToken || options.azureDeployment);\n\n    if (options.apiKey === '' && !isAzure) {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environment variable',\n      );\n    }\n\n    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;\n\n    if (!apiKey && !isAzure) {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environment variable',\n      );\n    }\n\n    if (!options.baseURL && isAzure) {\n      const azureEndpoint = process.env.AZURE_OPENAI_ENDPOINT;\n      if (!azureEndpoint) {\n        throw new Error(\n          'Missing Azure endpoint. Please pass base_url or set AZURE_OPENAI_ENDPOINT environment variable.',\n        );\n      }\n      options.baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n    }\n\n    const { modalities: _, ...optionsWithoutModalities } = options;\n    this._options = {\n      ...DEFAULT_REALTIME_MODEL_OPTIONS,\n      ...optionsWithoutModalities,\n      baseURL: options.baseURL || BASE_URL,\n      apiKey,\n      isAzure,\n      model: options.model || DEFAULT_REALTIME_MODEL_OPTIONS.model,\n      modalities,\n    };\n  }\n\n  /**\n   * Create a RealtimeModel instance configured for Azure OpenAI Service.\n   *\n   * @param azureDeployment - The name of your Azure OpenAI deployment.\n   * @param azureEndpoint - The endpoint URL for your Azure OpenAI resource. If undefined, will attempt to read from the environment variable AZURE_OPENAI_ENDPOINT.\n   * @param apiVersion - API version to use with Azure OpenAI Service. If undefined, will attempt to read from the environment variable OPENAI_API_VERSION.\n   * @param apiKey - Azure OpenAI API key. If undefined, will attempt to read from the environment variable AZURE_OPENAI_API_KEY.\n   * @param entraToken - Azure Entra authentication token. Required if not using API key authentication.\n   * @param baseURL - Base URL for the API endpoint. If undefined, constructed from the azure_endpoint.\n   * @param voice - Voice setting for audio outputs. Defaults to \"alloy\".\n   * @param inputAudioTranscription - Options for transcribing input audio. Defaults to @see DEFAULT_INPUT_AUDIO_TRANSCRIPTION.\n   * @param turnDetection - Options for server-based voice activity detection (VAD). Defaults to @see DEFAULT_SERVER_VAD_OPTIONS.\n   * @param temperature - Sampling temperature for response generation. Defaults to @see DEFAULT_TEMPERATURE.\n   * @param speed - Speed of the audio output. Defaults to 1.0.\n   * @param maxResponseOutputTokens - Maximum number of tokens in the response. Defaults to @see DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS.\n   * @param maxSessionDuration - Maximum duration of the session in milliseconds. Defaults to @see DEFAULT_MAX_SESSION_DURATION.\n   *\n   * @returns A RealtimeModel instance configured for Azure OpenAI Service.\n   *\n   * @throws Error if required Azure parameters are missing or invalid.\n   */\n  static withAzure({\n    azureDeployment,\n    azureEndpoint,\n    apiVersion,\n    apiKey,\n    entraToken,\n    baseURL,\n    voice = 'alloy',\n    inputAudioTranscription = AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n    turnDetection = AZURE_DEFAULT_TURN_DETECTION,\n    temperature = 0.8,\n    speed,\n  }: {\n    azureDeployment: string;\n    azureEndpoint?: string;\n    apiVersion?: string;\n    apiKey?: string;\n    entraToken?: string;\n    baseURL?: string;\n    voice?: string;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    // TODO(shubhra): add inputAudioNoiseReduction\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    speed?: number;\n  }) {\n    apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;\n    if (!apiKey && !entraToken) {\n      throw new Error(\n        'Missing credentials. Please pass one of `apiKey`, `entraToken`, or the `AZURE_OPENAI_API_KEY` environment variable.',\n      );\n    }\n\n    apiVersion = apiVersion || process.env.OPENAI_API_VERSION;\n    if (!apiVersion) {\n      throw new Error(\n        'Must provide either the `apiVersion` argument or the `OPENAI_API_VERSION` environment variable',\n      );\n    }\n\n    if (!baseURL) {\n      azureEndpoint = azureEndpoint || process.env.AZURE_OPENAI_ENDPOINT;\n      if (!azureEndpoint) {\n        throw new Error(\n          'Missing Azure endpoint. Please pass the `azure_endpoint` parameter or set the `AZURE_OPENAI_ENDPOINT` environment variable.',\n        );\n      }\n      baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n    }\n\n    return new RealtimeModel({\n      voice,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      speed,\n      apiKey,\n      azureDeployment,\n      apiVersion,\n      entraToken,\n      baseURL,\n    });\n  }\n\n  session() {\n    return new RealtimeSession(this);\n  }\n\n  async close() {\n    return;\n  }\n}\n\nfunction processBaseURL({\n  baseURL,\n  model,\n  isAzure = false,\n  azureDeployment,\n  apiVersion,\n}: {\n  baseURL: string;\n  model: string;\n  isAzure: boolean;\n  azureDeployment?: string;\n  apiVersion?: string;\n}): string {\n  const url = new URL([baseURL, 'realtime'].join('/'));\n\n  if (url.protocol === 'https:') {\n    url.protocol = 'wss:';\n  }\n\n  // ensure \"/realtime\" is added if the path is empty OR \"/v1\"\n  if (!url.pathname || ['', '/v1', '/openai'].includes(url.pathname.replace(/\\/$/, ''))) {\n    url.pathname = url.pathname.replace(/\\/$/, '') + '/realtime';\n  } else {\n    url.pathname = url.pathname.replace(/\\/$/, '');\n  }\n\n  const queryParams: Record<string, string> = {};\n  if (isAzure) {\n    if (apiVersion) {\n      queryParams['api-version'] = apiVersion;\n    }\n    if (azureDeployment) {\n      queryParams['deployment'] = azureDeployment;\n    }\n  } else {\n    queryParams['model'] = model;\n  }\n\n  for (const [key, value] of Object.entries(queryParams)) {\n    url.searchParams.set(key, value);\n  }\n\n  return url.toString();\n}\n\n/**\n * A session for the OpenAI Realtime API.\n *\n * This class is used to interact with the OpenAI Realtime API.\n * It is responsible for sending events to the OpenAI Realtime API and receiving events from it.\n *\n * It exposes two more events:\n * - openai_server_event_received: expose the raw server events from the OpenAI Realtime API\n * - openai_client_event_queued: expose the raw client events sent to the OpenAI Realtime API\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n  private _tools: llm.ToolContext = {};\n  private remoteChatCtx: llm.RemoteChatContext = new llm.RemoteChatContext();\n  private messageChannel = new Queue<api_proto.ClientEvent>();\n  private inputResampler?: AudioResampler;\n  private instructions?: string;\n  private oaiRealtimeModel: RealtimeModel;\n  private currentGeneration?: ResponseGeneration;\n  private responseCreatedFutures: { [id: string]: CreateResponseHandle } = {};\n\n  private textModeRecoveryRetries: number = 0;\n\n  private itemCreateFutures: { [id: string]: Future } = {};\n  private itemDeleteFutures: { [id: string]: Future } = {};\n\n  private updateChatCtxLock = new Mutex();\n  private updateFuncCtxLock = new Mutex();\n\n  // 100ms chunks\n  private bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS, SAMPLE_RATE / 10);\n\n  private pushedDurationMs: number = 0;\n\n  #logger = log();\n  #task: Task<void>;\n  #closed = false;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super(realtimeModel);\n\n    this.oaiRealtimeModel = realtimeModel;\n\n    this.#task = Task.from(({ signal }) => this.#mainTask(signal));\n\n    this.sendEvent(this.createSessionUpdateEvent());\n  }\n\n  sendEvent(command: api_proto.ClientEvent): void {\n    this.messageChannel.put(command);\n  }\n\n  private createSessionUpdateEvent(): api_proto.SessionUpdateEvent {\n    // OpenAI supports ['text'] or ['text', 'audio'] (audio always includes text transcript)\n    // We normalize to ensure 'text' is always present when using audio\n    const modalities: Modality[] = this.oaiRealtimeModel._options.modalities.includes('audio')\n      ? ['text', 'audio']\n      : ['text'];\n\n    return {\n      type: 'session.update',\n      session: {\n        model: this.oaiRealtimeModel._options.model,\n        voice: this.oaiRealtimeModel._options.voice,\n        input_audio_format: 'pcm16',\n        output_audio_format: 'pcm16',\n        modalities: modalities,\n        turn_detection: this.oaiRealtimeModel._options.turnDetection,\n        input_audio_transcription: this.oaiRealtimeModel._options.inputAudioTranscription,\n        // TODO(shubhra): add inputAudioNoiseReduction\n        temperature: this.oaiRealtimeModel._options.temperature,\n        tool_choice: toOaiToolChoice(this.oaiRealtimeModel._options.toolChoice),\n        max_response_output_tokens:\n          this.oaiRealtimeModel._options.maxResponseOutputTokens === Infinity\n            ? 'inf'\n            : this.oaiRealtimeModel._options.maxResponseOutputTokens,\n        // TODO(shubhra): add tracing options\n        instructions: this.instructions,\n        speed: this.oaiRealtimeModel._options.speed,\n      },\n    };\n  }\n\n  get chatCtx() {\n    return this.remoteChatCtx.toChatCtx();\n  }\n\n  get tools() {\n    return { ...this._tools } as llm.ToolContext;\n  }\n\n  async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {\n    const unlock = await this.updateChatCtxLock.lock();\n    const events = this.createChatCtxUpdateEvents(_chatCtx);\n    const futures: Future<void>[] = [];\n\n    for (const event of events) {\n      const future = new Future<void>();\n      futures.push(future);\n\n      if (event.type === 'conversation.item.create') {\n        this.itemCreateFutures[event.item.id] = future;\n      } else if (event.type == 'conversation.item.delete') {\n        this.itemDeleteFutures[event.item_id] = future;\n      }\n\n      this.sendEvent(event);\n    }\n\n    if (futures.length === 0) {\n      unlock();\n      return;\n    }\n\n    try {\n      // wait for futures to resolve or timeout\n      await Promise.race([\n        Promise.all(futures),\n        delay(5000).then(() => {\n          throw new Error('Chat ctx update events timed out');\n        }),\n      ]);\n    } catch (e) {\n      this.#logger.error((e as Error).message);\n      throw e;\n    } finally {\n      unlock();\n    }\n  }\n\n  private createChatCtxUpdateEvents(\n    chatCtx: llm.ChatContext,\n    addMockAudio: boolean = false,\n  ): (api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[] {\n    const newChatCtx = chatCtx.copy();\n    if (addMockAudio) {\n      newChatCtx.items.push(createMockAudioItem());\n    } else {\n      // clean up existing mock audio items\n      newChatCtx.items = newChatCtx.items.filter(\n        (item) => !item.id.startsWith(MOCK_AUDIO_ID_PREFIX),\n      );\n    }\n\n    const events: (\n      | api_proto.ConversationItemCreateEvent\n      | api_proto.ConversationItemDeleteEvent\n    )[] = [];\n\n    const diffOps = llm.computeChatCtxDiff(this.chatCtx, newChatCtx);\n    for (const op of diffOps.toRemove) {\n      events.push({\n        type: 'conversation.item.delete',\n        item_id: op,\n        event_id: shortuuid('chat_ctx_delete_'),\n      } as api_proto.ConversationItemDeleteEvent);\n    }\n\n    for (const [previousId, id] of diffOps.toCreate) {\n      const chatItem = newChatCtx.getById(id);\n      if (!chatItem) {\n        throw new Error(`Chat item ${id} not found`);\n      }\n      events.push({\n        type: 'conversation.item.create',\n        item: livekitItemToOpenAIItem(chatItem),\n        previous_item_id: previousId ?? undefined,\n        event_id: shortuuid('chat_ctx_create_'),\n      } as api_proto.ConversationItemCreateEvent);\n    }\n    return events;\n  }\n\n  async updateTools(_tools: llm.ToolContext): Promise<void> {\n    const unlock = await this.updateFuncCtxLock.lock();\n    const ev = this.createToolsUpdateEvent(_tools);\n    this.sendEvent(ev);\n\n    if (!ev.session.tools) {\n      throw new Error('Tools are missing in the session update event');\n    }\n\n    // TODO(brian): these logics below are noops I think, leaving it here to keep\n    // parity with the python but we should remove them later\n    const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name));\n    const retainedTools = Object.fromEntries(\n      Object.entries(_tools).filter(\n        ([name, tool]) => llm.isFunctionTool(tool) && retainedToolNames.has(name),\n      ),\n    );\n\n    this._tools = retainedTools as llm.ToolContext;\n\n    unlock();\n  }\n\n  private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent {\n    const oaiTools: api_proto.Tool[] = [];\n\n    for (const [name, tool] of Object.entries(_tools)) {\n      if (!llm.isFunctionTool(tool)) {\n        this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n        continue;\n      }\n\n      const { parameters: toolParameters, description } = tool;\n      try {\n        const parameters = llm.toJsonSchema(\n          toolParameters,\n        ) as unknown as api_proto.Tool['parameters'];\n\n        oaiTools.push({\n          name,\n          description,\n          parameters: parameters,\n          type: 'function',\n        });\n      } catch (e) {\n        this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n        continue;\n      }\n    }\n\n    return {\n      type: 'session.update',\n      session: {\n        model: this.oaiRealtimeModel._options.model,\n        tools: oaiTools,\n      },\n      event_id: shortuuid('tools_update_'),\n    };\n  }\n\n  async updateInstructions(_instructions: string): Promise<void> {\n    const eventId = shortuuid('instructions_update_');\n    this.sendEvent({\n      type: 'session.update',\n      session: {\n        instructions: _instructions,\n      },\n      event_id: eventId,\n    } as api_proto.SessionUpdateEvent);\n    this.instructions = _instructions;\n  }\n\n  updateOptions({ toolChoice }: { toolChoice?: llm.ToolChoice }): void {\n    const options: api_proto.SessionUpdateEvent['session'] = {};\n\n    this.oaiRealtimeModel._options.toolChoice = toolChoice;\n    options.tool_choice = toOaiToolChoice(toolChoice);\n\n    // TODO(brian): add other options here\n\n    this.sendEvent({\n      type: 'session.update',\n      session: options,\n      event_id: shortuuid('options_update_'),\n    });\n  }\n\n  pushAudio(frame: AudioFrame): void {\n    for (const f of this.resampleAudio(frame)) {\n      for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n        this.sendEvent({\n          type: 'input_audio_buffer.append',\n          audio: Buffer.from(nf.data.buffer).toString('base64'),\n        } as api_proto.InputAudioBufferAppendEvent);\n        // TODO(AJS-102): use frame.durationMs once available in rtc-node\n        this.pushedDurationMs += (nf.samplesPerChannel / nf.sampleRate) * 1000;\n      }\n    }\n  }\n\n  async commitAudio(): Promise<void> {\n    if (this.pushedDurationMs > 100) {\n      // OpenAI requires at least 100ms of audio\n      this.sendEvent({\n        type: 'input_audio_buffer.commit',\n      } as api_proto.InputAudioBufferCommitEvent);\n      this.pushedDurationMs = 0;\n    }\n  }\n\n  async clearAudio(): Promise<void> {\n    this.sendEvent({\n      type: 'input_audio_buffer.clear',\n    } as api_proto.InputAudioBufferClearEvent);\n    this.pushedDurationMs = 0;\n  }\n\n  async generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent> {\n    const handle = this.createResponse({ instructions, userInitiated: true });\n    this.textModeRecoveryRetries = 0;\n    return handle.doneFut.await;\n  }\n\n  async interrupt(): Promise<void> {\n    this.sendEvent({\n      type: 'response.cancel',\n    } as api_proto.ResponseCancelEvent);\n  }\n\n  async truncate(_options: {\n    messageId: string;\n    audioEndMs: number;\n    modalities?: Modality[];\n    audioTranscript?: string;\n  }): Promise<void> {\n    if (!_options.modalities || _options.modalities.includes('audio')) {\n      this.sendEvent({\n        type: 'conversation.item.truncate',\n        content_index: 0,\n        item_id: _options.messageId,\n        audio_end_ms: _options.audioEndMs,\n      } as api_proto.ConversationItemTruncateEvent);\n    } else if (_options.audioTranscript !== undefined) {\n      // sync it to the remote chat context\n      const chatCtx = this.chatCtx.copy();\n      const idx = chatCtx.indexById(_options.messageId);\n      if (idx !== undefined) {\n        const item = chatCtx.items[idx];\n        if (item && item.type === 'message') {\n          const newItem = llm.ChatMessage.create({\n            ...item,\n            content: [_options.audioTranscript],\n          });\n          chatCtx.items[idx] = newItem;\n          const events = this.createChatCtxUpdateEvents(chatCtx);\n          for (const ev of events) {\n            this.sendEvent(ev);\n          }\n        }\n      }\n    }\n  }\n\n  private loggableEvent(\n    event: api_proto.ClientEvent | api_proto.ServerEvent,\n  ): Record<string, unknown> {\n    const untypedEvent: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(event)) {\n      if (value !== undefined) {\n        untypedEvent[key] = value;\n      }\n    }\n\n    if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n      return { ...untypedEvent, audio: '...' };\n    }\n    if (\n      untypedEvent.delta &&\n      typeof untypedEvent.delta === 'string' &&\n      event.type === 'response.audio.delta'\n    ) {\n      return { ...untypedEvent, delta: '...' };\n    }\n    return untypedEvent;\n  }\n\n  private async createWsConn(): Promise<WebSocket> {\n    const headers: Record<string, string> = {\n      'User-Agent': 'LiveKit-Agents-JS',\n    };\n\n    if (this.oaiRealtimeModel._options.isAzure) {\n      // Microsoft API has two ways of authentication\n      // 1. Entra token set as `Bearer` token\n      // 2. API key set as `api_key` header (also accepts query string)\n      if (this.oaiRealtimeModel._options.entraToken) {\n        headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.entraToken}`;\n      } else if (this.oaiRealtimeModel._options.apiKey) {\n        headers['api-key'] = this.oaiRealtimeModel._options.apiKey;\n      } else {\n        throw new Error('Microsoft API key or entraToken is required');\n      }\n    } else {\n      headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.apiKey}`;\n      headers['OpenAI-Beta'] = 'realtime=v1';\n    }\n\n    const url = processBaseURL({\n      baseURL: this.oaiRealtimeModel._options.baseURL,\n      model: this.oaiRealtimeModel._options.model,\n      isAzure: this.oaiRealtimeModel._options.isAzure,\n      apiVersion: this.oaiRealtimeModel._options.apiVersion,\n      azureDeployment: this.oaiRealtimeModel._options.azureDeployment,\n    });\n\n    if (lkOaiDebug) {\n      this.#logger.debug(`Connecting to OpenAI Realtime API at ${url}`);\n    }\n\n    return new Promise((resolve, reject) => {\n      const ws = new WebSocket(url, { headers });\n      let waiting = true;\n\n      const timeout = setTimeout(() => {\n        ws.close();\n        reject(new Error('WebSocket connection timeout'));\n      }, this.oaiRealtimeModel._options.connOptions.timeoutMs);\n\n      ws.once('open', () => {\n        if (!waiting) return;\n        waiting = false;\n        clearTimeout(timeout);\n        resolve(ws);\n      });\n\n      ws.once('close', () => {\n        if (!waiting) return;\n        waiting = false;\n        clearTimeout(timeout);\n        reject(new Error('OpenAI Realtime API connection closed'));\n      });\n    });\n  }\n\n  async #mainTask(signal: AbortSignal): Promise<void> {\n    let reconnecting = false;\n    let numRetries = 0;\n    let wsConn: WebSocket | null = null;\n    const maxRetries = this.oaiRealtimeModel._options.connOptions.maxRetry;\n\n    const reconnect = async () => {\n      this.#logger.debug(\n        {\n          maxSessionDuration: this.oaiRealtimeModel._options.maxSessionDuration,\n        },\n        'Reconnecting to OpenAI Realtime API',\n      );\n\n      const events: api_proto.ClientEvent[] = [];\n\n      // options and instructions\n      events.push(this.createSessionUpdateEvent());\n\n      // tools\n      if (Object.keys(this._tools).length > 0) {\n        events.push(this.createToolsUpdateEvent(this._tools));\n      }\n\n      // chat context\n      const chatCtx = this.chatCtx.copy({\n        excludeFunctionCall: true,\n        excludeInstructions: true,\n        excludeEmptyMessage: true,\n      });\n\n      const oldChatCtx = this.remoteChatCtx;\n      this.remoteChatCtx = new llm.RemoteChatContext();\n      events.push(...this.createChatCtxUpdateEvents(chatCtx));\n\n      try {\n        for (const ev of events) {\n          this.emit('openai_client_event_queued', ev);\n          wsConn!.send(JSON.stringify(ev));\n        }\n      } catch (error) {\n        this.remoteChatCtx = oldChatCtx;\n        throw new APIConnectionError({\n          message: 'Failed to send message to OpenAI Realtime API during session re-connection',\n        });\n      }\n\n      this.#logger.debug('Reconnected to OpenAI Realtime API');\n\n      this.emit('session_reconnected', {} as llm.RealtimeSessionReconnectedEvent);\n    };\n\n    reconnecting = false;\n    while (!this.#closed && !signal.aborted) {\n      this.#logger.debug('Creating WebSocket connection to OpenAI Realtime API');\n      wsConn = await this.createWsConn();\n      if (signal.aborted) break;\n\n      try {\n        if (reconnecting) {\n          await reconnect();\n          if (signal.aborted) break;\n          numRetries = 0;\n        }\n\n        await this.runWs(wsConn);\n        if (signal.aborted) break;\n      } catch (error) {\n        if (!isAPIError(error)) {\n          this.emitError({ error: error as Error, recoverable: false });\n          throw error;\n        }\n\n        if (maxRetries === 0 || !error.retryable) {\n          this.emitError({ error: error as Error, recoverable: false });\n          throw error;\n        }\n\n        if (numRetries === maxRetries) {\n          this.emitError({ error: error as Error, recoverable: false });\n          throw new APIConnectionError({\n            message: `OpenAI Realtime API connection failed after ${numRetries} attempts`,\n            options: {\n              body: error,\n              retryable: false,\n            },\n          });\n        }\n\n        this.emitError({ error: error as Error, recoverable: true });\n        const retryInterval =\n          numRetries === 0\n            ? DEFAULT_FIRST_RETRY_INTERVAL_MS\n            : this.oaiRealtimeModel._options.connOptions.retryIntervalMs;\n        this.#logger.warn(\n          {\n            attempt: numRetries,\n            maxRetries,\n            error,\n          },\n          `OpenAI Realtime API connection failed, retrying in ${retryInterval / 1000}s`,\n        );\n\n        await delay(retryInterval);\n        numRetries++;\n      }\n\n      reconnecting = true;\n    }\n  }\n\n  private async runWs(wsConn: WebSocket): Promise<void> {\n    const forwardEvents = async (signal: AbortSignal): Promise<void> => {\n      const abortFuture = new Future<void>();\n      signal.addEventListener('abort', () => abortFuture.resolve());\n\n      while (!this.#closed && wsConn.readyState === WebSocket.OPEN && !signal.aborted) {\n        try {\n          const event = await Promise.race([this.messageChannel.get(), abortFuture.await]);\n          if (signal.aborted || abortFuture.done || event === undefined) {\n            break;\n          }\n\n          if (lkOaiDebug) {\n            this.#logger.debug(this.loggableEvent(event), `(client) -> ${event.type}`);\n          }\n\n          this.emit('openai_client_event_queued', event);\n          wsConn.send(JSON.stringify(event));\n        } catch (error) {\n          break;\n        }\n      }\n\n      wsConn.close();\n    };\n\n    const wsCloseFuture = new Future<void | Error>();\n\n    wsConn.onerror = (error) => {\n      wsCloseFuture.resolve(new APIConnectionError({ message: error.message }));\n    };\n    wsConn.onclose = () => {\n      wsCloseFuture.resolve();\n    };\n\n    wsConn.onmessage = (message: MessageEvent) => {\n      const event: api_proto.ServerEvent = JSON.parse(message.data as string);\n\n      this.emit('openai_server_event_received', event);\n      if (lkOaiDebug) {\n        this.#logger.debug(this.loggableEvent(event), `(server) <- ${event.type}`);\n      }\n\n      switch (event.type) {\n        case 'input_audio_buffer.speech_started':\n          this.handleInputAudioBufferSpeechStarted(event);\n          break;\n        case 'input_audio_buffer.speech_stopped':\n          this.handleInputAudioBufferSpeechStopped(event);\n          break;\n        case 'response.created':\n          this.handleResponseCreated(event);\n          break;\n        case 'response.output_item.added':\n          this.handleResponseOutputItemAdded(event);\n          break;\n        case 'conversation.item.created':\n          this.handleConversationItemCreated(event);\n          break;\n        case 'conversation.item.deleted':\n          this.handleConversationItemDeleted(event);\n          break;\n        case 'conversation.item.input_audio_transcription.completed':\n          this.handleConversationItemInputAudioTranscriptionCompleted(event);\n          break;\n        case 'conversation.item.input_audio_transcription.failed':\n          this.handleConversationItemInputAudioTranscriptionFailed(event);\n          break;\n        case 'response.content_part.added':\n          this.handleResponseContentPartAdded(event);\n          break;\n        case 'response.content_part.done':\n          this.handleResponseContentPartDone(event);\n          break;\n        case 'response.text.delta':\n          this.handleResponseTextDelta(event);\n          break;\n        case 'response.text.done':\n          this.handleResponseTextDone(event);\n          break;\n        case 'response.audio_transcript.delta':\n          this.handleResponseAudioTranscriptDelta(event);\n          break;\n        case 'response.audio.delta':\n          this.handleResponseAudioDelta(event);\n          break;\n        case 'response.audio_transcript.done':\n          this.handleResponseAudioTranscriptDone(event);\n          break;\n        case 'response.audio.done':\n          this.handleResponseAudioDone(event);\n          break;\n        case 'response.output_item.done':\n          this.handleResponseOutputItemDone(event);\n          break;\n        case 'response.done':\n          this.handleResponseDone(event);\n          break;\n        case 'error':\n          this.handleError(event);\n          break;\n        default:\n          if (lkOaiDebug) {\n            this.#logger.debug(`unhandled event: ${event.type}`);\n          }\n          break;\n      }\n    };\n\n    const sendTask = Task.from(({ signal }) => forwardEvents(signal));\n\n    const wsTask = Task.from(({ signal }) => {\n      const abortPromise = new Promise<void>((resolve) => {\n        signal.addEventListener('abort', () => {\n          resolve();\n        });\n      });\n\n      return Promise.race([wsCloseFuture.await, abortPromise]);\n    });\n\n    const waitReconnectTask = Task.from(async ({ signal }) => {\n      await delay(this.oaiRealtimeModel._options.maxSessionDuration, { signal });\n      return new APIConnectionError({\n        message: 'OpenAI Realtime API connection timeout',\n      });\n    });\n\n    try {\n      const result = await Promise.race([wsTask.result, sendTask.result, waitReconnectTask.result]);\n\n      if (waitReconnectTask.done && this.currentGeneration) {\n        await this.currentGeneration._doneFut.await;\n      }\n\n      if (result instanceof Error) {\n        throw result;\n      }\n    } finally {\n      await cancelAndWait([wsTask, sendTask, waitReconnectTask], 2000);\n      wsConn.close();\n    }\n  }\n\n  async close() {\n    super.close();\n    this.#closed = true;\n    await this.#task;\n  }\n\n  private handleInputAudioBufferSpeechStarted(\n    _event: api_proto.InputAudioBufferSpeechStartedEvent,\n  ): void {\n    this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n  }\n\n  private handleInputAudioBufferSpeechStopped(\n    _event: api_proto.InputAudioBufferSpeechStoppedEvent,\n  ): void {\n    this.emit('input_speech_stopped', {\n      userTranscriptionEnabled: this.oaiRealtimeModel._options.inputAudioTranscription !== null,\n    } as llm.InputSpeechStoppedEvent);\n  }\n\n  private handleResponseCreated(event: api_proto.ResponseCreatedEvent): void {\n    if (!event.response.id) {\n      throw new Error('response.id is missing');\n    }\n\n    this.currentGeneration = {\n      messageChannel: stream.createStreamChannel<llm.MessageGeneration>(),\n      functionChannel: stream.createStreamChannel<llm.FunctionCall>(),\n      messages: new Map(),\n      _doneFut: new Future(),\n      _createdTimestamp: Date.now(),\n    };\n\n    // Build generation event and resolve client future (if any) before emitting,\n    // matching Python behavior.\n    const generationEv = {\n      messageStream: this.currentGeneration.messageChannel.stream(),\n      functionStream: this.currentGeneration.functionChannel.stream(),\n      userInitiated: false,\n      responseId: event.response.id,\n    } as llm.GenerationCreatedEvent;\n\n    const clientEventId = event.response.metadata?.client_event_id;\n    if (clientEventId) {\n      const handle = this.responseCreatedFutures[clientEventId];\n      if (handle) {\n        delete this.responseCreatedFutures[clientEventId];\n        generationEv.userInitiated = true;\n        if (!handle.doneFut.done) {\n          handle.doneFut.resolve(generationEv);\n        }\n      }\n    }\n\n    this.emit('generation_created', generationEv);\n  }\n\n  private handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    if (!event.item.type) {\n      throw new Error('item.type is not set');\n    }\n\n    if (!event.response_id) {\n      throw new Error('response_id is not set');\n    }\n\n    const itemType = event.item.type;\n    const responseId = event.response_id;\n\n    if (itemType !== 'message') {\n      // emit immediately if it's not a message, otherwise wait response.content_part.added\n      this.resolveGeneration(responseId);\n      this.textModeRecoveryRetries = 0;\n      return;\n    }\n\n    const itemId = event.item.id;\n    if (!itemId) {\n      throw new Error('item.id is not set');\n    }\n\n    const modalitiesFut = new Future<Modality[]>();\n    const itemGeneration: MessageGeneration = {\n      messageId: itemId,\n      textChannel: stream.createStreamChannel<string>(),\n      audioChannel: stream.createStreamChannel<AudioFrame>(),\n      audioTranscript: '',\n      modalities: modalitiesFut,\n    };\n\n    // If audioOutput is not supported, close audio channel immediately\n    if (!this.oaiRealtimeModel.capabilities.audioOutput) {\n      itemGeneration.audioChannel.close();\n      modalitiesFut.resolve(['text']);\n    }\n\n    this.currentGeneration.messageChannel.write({\n      messageId: itemId,\n      textStream: itemGeneration.textChannel.stream(),\n      audioStream: itemGeneration.audioChannel.stream(),\n      modalities: modalitiesFut.await,\n    });\n\n    this.currentGeneration.messages.set(itemId, itemGeneration);\n  }\n\n  private handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {\n    if (!event.item.id) {\n      throw new Error('item.id is not set');\n    }\n\n    try {\n      this.remoteChatCtx.insert(event.previous_item_id, openAIItemToLivekitItem(event.item));\n    } catch (error) {\n      this.#logger.error({ error, itemId: event.item.id }, 'failed to insert conversation item');\n    }\n\n    const fut = this.itemCreateFutures[event.item.id];\n    if (fut) {\n      fut.resolve();\n      delete this.itemCreateFutures[event.item.id];\n    }\n  }\n\n  private handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {\n    if (!event.item_id) {\n      throw new Error('item_id is not set');\n    }\n\n    try {\n      this.remoteChatCtx.delete(event.item_id);\n    } catch (error) {\n      this.#logger.error({ error, itemId: event.item_id }, 'failed to delete conversation item');\n    }\n\n    const fut = this.itemDeleteFutures[event.item_id];\n    if (fut) {\n      fut.resolve();\n      delete this.itemDeleteFutures[event.item_id];\n    }\n  }\n\n  private handleConversationItemInputAudioTranscriptionCompleted(\n    event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n  ): void {\n    const remoteItem = this.remoteChatCtx.get(event.item_id);\n    if (!remoteItem) {\n      return;\n    }\n\n    const item = remoteItem.item;\n    if (item instanceof llm.ChatMessage) {\n      item.content.push(event.transcript);\n    } else {\n      throw new Error('item is not a chat message');\n    }\n\n    this.emit('input_audio_transcription_completed', {\n      itemId: event.item_id,\n      transcript: event.transcript,\n      isFinal: true,\n    } as llm.InputTranscriptionCompleted);\n  }\n\n  private handleConversationItemInputAudioTranscriptionFailed(\n    event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n  ): void {\n    this.#logger.error(\n      { error: event.error },\n      'OpenAI Realtime API failed to transcribe input audio',\n    );\n  }\n\n  private handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemId = event.item_id;\n    const itemType = event.part.type;\n\n    const itemGeneration = this.currentGeneration.messages.get(itemId);\n    if (!itemGeneration) {\n      this.#logger.warn(`itemGeneration not found for itemId=${itemId}`);\n      return;\n    }\n\n    if (itemType === 'text' && this.oaiRealtimeModel.capabilities.audioOutput) {\n      this.#logger.warn('Text response received from OpenAI Realtime API in audio modality.');\n    }\n\n    if (!itemGeneration.modalities.done) {\n      const modalityResult: Modality[] = itemType === 'text' ? ['text'] : ['audio', 'text'];\n      itemGeneration.modalities.resolve(modalityResult);\n    }\n\n    if (this.currentGeneration._firstTokenTimestamp === undefined) {\n      this.currentGeneration._firstTokenTimestamp = Date.now();\n    }\n  }\n\n  private handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n    if (!event.part) {\n      return;\n    }\n    if (event.part.type !== 'text') {\n      return;\n    }\n\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    // TODO(shubhra): handle text mode recovery\n  }\n\n  private handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemGeneration = this.currentGeneration.messages.get(event.item_id);\n    if (!itemGeneration) {\n      throw new Error('itemGeneration is not set');\n    }\n\n    if (\n      !this.oaiRealtimeModel.capabilities.audioOutput &&\n      !this.currentGeneration._firstTokenTimestamp\n    ) {\n      this.currentGeneration._firstTokenTimestamp = Date.now();\n    }\n\n    itemGeneration.textChannel.write(event.delta);\n    itemGeneration.audioTranscript += event.delta;\n  }\n\n  private handleResponseTextDone(_event: api_proto.ResponseTextDoneEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n  }\n\n  private handleResponseAudioTranscriptDelta(\n    event: api_proto.ResponseAudioTranscriptDeltaEvent,\n  ): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemId = event.item_id;\n    const delta = event.delta;\n\n    // TODO (shubhra): add timed string support\n\n    const itemGeneration = this.currentGeneration.messages.get(itemId);\n    if (!itemGeneration) {\n      throw new Error('itemGeneration is not set');\n    } else {\n      itemGeneration.textChannel.write(delta);\n      itemGeneration.audioTranscript += delta;\n    }\n  }\n\n  private handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemGeneration = this.currentGeneration.messages.get(event.item_id);\n    if (!itemGeneration) {\n      throw new Error('itemGeneration is not set');\n    }\n\n    if (this.currentGeneration._firstTokenTimestamp === undefined) {\n      this.currentGeneration._firstTokenTimestamp = Date.now();\n    }\n\n    if (!itemGeneration.modalities.done) {\n      itemGeneration.modalities.resolve(['audio', 'text']);\n    }\n\n    const binaryString = atob(event.delta);\n    const len = binaryString.length;\n    const bytes = new Uint8Array(len);\n    for (let i = 0; i < len; i++) {\n      bytes[i] = binaryString.charCodeAt(i);\n    }\n\n    itemGeneration.audioChannel.write(\n      new AudioFrame(\n        new Int16Array(bytes.buffer),\n        api_proto.SAMPLE_RATE,\n        api_proto.NUM_CHANNELS,\n        bytes.length / 2,\n      ),\n    );\n  }\n\n  private handleResponseAudioTranscriptDone(\n    _event: api_proto.ResponseAudioTranscriptDoneEvent,\n  ): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n  }\n\n  private handleResponseAudioDone(_event: api_proto.ResponseAudioDoneEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n  }\n\n  private handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const itemId = event.item.id;\n    const itemType = event.item.type;\n\n    if (itemType === 'function_call') {\n      const item = event.item;\n      if (!item.call_id || !item.name || !item.arguments) {\n        throw new Error('item is not a function call');\n      }\n      this.currentGeneration.functionChannel.write(\n        llm.FunctionCall.create({\n          callId: item.call_id,\n          name: item.name,\n          args: item.arguments,\n        }),\n      );\n    } else if (itemType === 'message') {\n      const itemGeneration = this.currentGeneration.messages.get(itemId);\n      if (!itemGeneration) {\n        return;\n      }\n      // text response doesn't have itemGeneration\n      itemGeneration.textChannel.close();\n      itemGeneration.audioChannel.close();\n      if (!itemGeneration.modalities.done) {\n        // In case message modalities is not set, this shouldn't happen\n        itemGeneration.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n      }\n    }\n  }\n\n  private handleResponseDone(_event: api_proto.ResponseDoneEvent): void {\n    if (!this.currentGeneration) {\n      // OpenAI has a race condition where we could receive response.done without any\n      // previous response.created (This happens generally during interruption)\n      return;\n    }\n\n    const createdTimestamp = this.currentGeneration._createdTimestamp;\n    const firstTokenTimestamp = this.currentGeneration._firstTokenTimestamp;\n\n    this.#logger.debug(\n      {\n        messageCount: this.currentGeneration.messages.size,\n      },\n      'Closing generation channels in handleResponseDone',\n    );\n\n    for (const generation of this.currentGeneration.messages.values()) {\n      generation.textChannel.close();\n      generation.audioChannel.close();\n      if (!generation.modalities.done) {\n        generation.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n      }\n    }\n\n    this.currentGeneration.functionChannel.close();\n    this.currentGeneration.messageChannel.close();\n\n    for (const itemId of this.currentGeneration.messages.keys()) {\n      const remoteItem = this.remoteChatCtx.get(itemId);\n      if (remoteItem && remoteItem.item instanceof llm.ChatMessage) {\n        remoteItem.item.content.push(this.currentGeneration.messages.get(itemId)!.audioTranscript);\n      }\n    }\n\n    this.currentGeneration._doneFut.resolve();\n    this.currentGeneration = undefined;\n\n    // Calculate and emit metrics\n    const usage = _event.response.usage;\n    const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n    const durationMs = Date.now() - createdTimestamp;\n\n    const realtimeMetrics: metrics.RealtimeModelMetrics = {\n      type: 'realtime_model_metrics',\n      timestamp: createdTimestamp,\n      requestId: _event.response.id || '',\n      ttftMs,\n      durationMs,\n      cancelled: _event.response.status === 'cancelled',\n      label: 'openai_realtime',\n      inputTokens: usage?.input_tokens ?? 0,\n      outputTokens: usage?.output_tokens ?? 0,\n      totalTokens: usage?.total_tokens ?? 0,\n      tokensPerSecond: durationMs > 0 ? (usage?.output_tokens ?? 0) / (durationMs / 1000) : 0,\n      inputTokenDetails: {\n        audioTokens: usage?.input_token_details?.audio_tokens ?? 0,\n        textTokens: usage?.input_token_details?.text_tokens ?? 0,\n        imageTokens: 0, // Not supported yet\n        cachedTokens: usage?.input_token_details?.cached_tokens ?? 0,\n        cachedTokensDetails: usage?.input_token_details?.cached_tokens_details\n          ? {\n              audioTokens: usage?.input_token_details?.cached_tokens_details?.audio_tokens ?? 0,\n              textTokens: usage?.input_token_details?.cached_tokens_details?.text_tokens ?? 0,\n              imageTokens: usage?.input_token_details?.cached_tokens_details?.image_tokens ?? 0,\n            }\n          : undefined,\n      },\n      outputTokenDetails: {\n        textTokens: usage?.output_token_details?.text_tokens ?? 0,\n        audioTokens: usage?.output_token_details?.audio_tokens ?? 0,\n        imageTokens: 0,\n      },\n    };\n\n    this.emit('metrics_collected', realtimeMetrics);\n    // TODO(brian): handle response done but not complete\n  }\n\n  private handleError(event: api_proto.ErrorEvent): void {\n    if (event.error.message.startsWith('Cancellation failed')) {\n      return;\n    }\n\n    this.#logger.error({ error: event.error }, 'OpenAI Realtime API returned an error');\n    this.emitError({\n      error: new APIError(event.error.message, {\n        body: event.error,\n        retryable: true,\n      }),\n      recoverable: true,\n    });\n\n    // TODO(brian): set error for response future if it exists\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }): void {\n    // IMPORTANT: only emit error if there are listeners; otherwise emit will throw an error\n    this.emit('error', {\n      timestamp: Date.now(),\n      // TODO(brian): add label\n      label: '',\n      error,\n      recoverable,\n    } as llm.RealtimeModelError);\n  }\n\n  private *resampleAudio(frame: AudioFrame): Generator<AudioFrame> {\n    yield frame;\n  }\n\n  private createResponse({\n    userInitiated,\n    instructions,\n    oldHandle,\n  }: {\n    userInitiated: boolean;\n    instructions?: string;\n    oldHandle?: CreateResponseHandle;\n  }): CreateResponseHandle {\n    const handle = oldHandle || new CreateResponseHandle({ instructions });\n    if (oldHandle && instructions) {\n      handle.instructions = instructions;\n    }\n\n    const eventId = shortuuid('response_create_');\n    if (userInitiated) {\n      this.responseCreatedFutures[eventId] = handle;\n    }\n\n    const response: api_proto.ResponseCreateEvent['response'] = {};\n    if (instructions) response.instructions = instructions;\n    if (userInitiated) response.metadata = { client_event_id: eventId };\n\n    this.sendEvent({\n      type: 'response.create',\n      event_id: eventId,\n      response: Object.keys(response).length > 0 ? response : undefined,\n    });\n\n    return handle;\n  }\n\n  private resolveGeneration(responseId: string): void {\n    if (!this.currentGeneration) {\n      throw new Error('currentGeneration is not set');\n    }\n\n    const generation_ev = {\n      messageStream: this.currentGeneration.messageChannel.stream(),\n      functionStream: this.currentGeneration.functionChannel.stream(),\n      userInitiated: false,\n      responseId,\n    } as llm.GenerationCreatedEvent;\n\n    const handle = this.responseCreatedFutures[responseId];\n    if (handle) {\n      delete this.responseCreatedFutures[responseId];\n      generation_ev.userInitiated = true;\n      if (handle.doneFut.done) {\n        this.#logger.warn({ responseId }, 'response received after timeout');\n      } else {\n        handle.doneFut.resolve(generation_ev);\n      }\n    }\n  }\n}\n\nfunction livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {\n  switch (item.type) {\n    case 'function_call':\n      return {\n        id: item.id,\n        type: 'function_call',\n        call_id: item.callId,\n        name: item.name,\n        arguments: item.args,\n      } as api_proto.FunctionCallItem;\n    case 'function_call_output':\n      return {\n        id: item.id,\n        type: 'function_call_output',\n        call_id: item.callId,\n        output: item.output,\n      } as api_proto.FunctionCallOutputItem;\n    case 'message':\n      const role = item.role === 'developer' ? 'system' : item.role;\n      const contentList: api_proto.Content[] = [];\n      for (const c of item.content) {\n        if (typeof c === 'string') {\n          contentList.push({\n            type: role === 'assistant' ? 'text' : 'input_text',\n            text: c,\n          } as api_proto.InputTextContent);\n        } else if (c.type === 'image_content') {\n          // not supported for now\n          continue;\n        } else if (c.type === 'audio_content') {\n          if (role === 'user') {\n            const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString('base64');\n            contentList.push({\n              type: 'input_audio',\n              audio: encodedAudio,\n            } as api_proto.InputAudioContent);\n          }\n        }\n      }\n      return {\n        id: item.id,\n        type: 'message',\n        role,\n        content: contentList,\n      } as api_proto.UserItem;\n    default:\n      throw new Error(`Unsupported item type: ${(item as any).type}`);\n  }\n}\n\nfunction openAIItemToLivekitItem(item: api_proto.ItemResource): llm.ChatItem {\n  if (!item.id) {\n    throw new Error('item.id is not set');\n  }\n\n  switch (item.type) {\n    case 'function_call':\n      return llm.FunctionCall.create({\n        id: item.id,\n        callId: item.call_id,\n        name: item.name,\n        args: item.arguments,\n      });\n    case 'function_call_output':\n      return llm.FunctionCallOutput.create({\n        id: item.id,\n        callId: item.call_id,\n        output: item.output,\n        isError: false,\n      });\n    case 'message':\n      const content: llm.ChatContent[] = [];\n      // item.content can be a single object or an array; normalize to array\n      const contents = Array.isArray(item.content) ? item.content : [item.content];\n      for (const c of contents) {\n        if (c.type === 'text' || c.type === 'input_text') {\n          content.push(c.text);\n        }\n      }\n      return llm.ChatMessage.create({\n        id: item.id,\n        role: item.role,\n        content,\n      });\n  }\n}\n\nfunction createMockAudioItem(durationSeconds: number = 2): llm.ChatMessage {\n  const audioData = Buffer.alloc(durationSeconds * SAMPLE_RATE);\n  return llm.ChatMessage.create({\n    id: shortuuid(MOCK_AUDIO_ID_PREFIX),\n    role: 'user',\n    content: [\n      {\n        type: 'audio_content',\n        frame: [\n          new AudioFrame(\n            new Int16Array(audioData.buffer),\n            SAMPLE_RATE,\n            NUM_CHANNELS,\n            audioData.length / 2,\n          ),\n        ],\n      } as llm.AudioContent,\n    ],\n  });\n}\n\nfunction toOaiToolChoice(toolChoice?: llm.ToolChoice): api_proto.ToolChoice {\n  if (typeof toolChoice === 'string') {\n    return toolChoice;\n  }\n\n  if (toolChoice?.type === 'function') {\n    return toolChoice.function.name;\n  }\n\n  return 'auto';\n}\n"],"mappings":"AAIA;AAAA,EAEE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AAEtB,SAAS,YAAY,0BAA0B;AAC/C,SAA4B,iBAAiB;AAC7C,YAAY,eAAe;AAG3B,MAAM,aAAa,QAAQ,IAAI,kBAAkB,OAAO,QAAQ,IAAI,eAAe,IAAI;AAEvF,MAAM,cAAc;AACpB,MAAM,eAAe;AACrB,MAAM,WAAW;AAEjB,MAAM,uBAAuB;AAgD7B,MAAM,qBAAqB;AAAA,EACzB;AAAA,EACA;AAAA;AAAA,EAEA,YAAY,EAAE,aAAa,GAA8B;AACvD,SAAK,eAAe;AACpB,SAAK,UAAU,IAAI,OAAO;AAAA,EAC5B;AACF;AAGA,MAAM,kCAAkC;AACxC,MAAM,sBAAsB;AAC5B,MAAM,yBAAsD;AAAA,EAC1D,MAAM;AAAA,EACN,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,oBAAoB;AACtB;AACA,MAAM,oCAAuE;AAAA,EAC3E,OAAO;AACT;AACA,MAAM,sBAAsC;AAC5C,MAAM,qCAAqD;AAE3D,MAAM,0CAA6E;AAAA,EACjF,OAAO;AACT;AAEA,MAAM,+BAA4D;AAAA,EAChE,MAAM;AAAA,EACN,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,iBAAiB;AACnB;AAEA,MAAM,+BAA+B,KAAK,KAAK;AAE/C,MAAM,iCAAiC;AAAA,EACrC,OAAO;AAAA,EACP,OAAO;AAAA,EACP,aAAa;AAAA,EACb,yBAAyB;AAAA,EACzB,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,yBAAyB;AAAA,EACzB,oBAAoB;AAAA,EACpB,aAAa;AAAA,EACb,YAAY,CAAC,QAAQ,OAAO;AAC9B;AACO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA;AAAA,EAGzB;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,YACE,UAkBI,CAAC,GACL;AACA,UAAM,aAAc,QAAQ,cAC1B,+BAA+B;AAEjC,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe,QAAQ,kBAAkB;AAAA,MACzC,mBAAmB,QAAQ,4BAA4B;AAAA,MACvD,yBAAyB;AAAA,MACzB,aAAa,WAAW,SAAS,OAAO;AAAA,IAC1C,CAAC;AAED,UAAM,UAAU,CAAC,EAAE,QAAQ,cAAc,QAAQ,cAAc,QAAQ;AAEvE,QAAI,QAAQ,WAAW,MAAM,CAAC,SAAS;AACrC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAE7C,QAAI,CAAC,UAAU,CAAC,SAAS;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,QAAQ,WAAW,SAAS;AAC/B,YAAM,gBAAgB,QAAQ,IAAI;AAClC,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,cAAQ,UAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IACvD;AAEA,UAAM,EAAE,YAAY,GAAG,GAAG,yBAAyB,IAAI;AACvD,SAAK,WAAW;AAAA,MACd,GAAG;AAAA,MACH,GAAG;AAAA,MACH,SAAS,QAAQ,WAAW;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,OAAO,QAAQ,SAAS,+BAA+B;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuBA,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR,0BAA0B;AAAA,IAC1B,gBAAgB;AAAA,IAChB,cAAc;AAAA,IACd;AAAA,EACF,GAaG;AACD,aAAS,UAAU,QAAQ,IAAI;AAC/B,QAAI,CAAC,UAAU,CAAC,YAAY;AAC1B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,iBAAa,cAAc,QAAQ,IAAI;AACvC,QAAI,CAAC,YAAY;AACf,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,SAAS;AACZ,sBAAgB,iBAAiB,QAAQ,IAAI;AAC7C,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,gBAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IAC/C;AAEA,WAAO,IAAI,cAAc;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA,EAEA,MAAM,QAAQ;AACZ;AAAA,EACF;AACF;AAEA,SAAS,eAAe;AAAA,EACtB;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV;AAAA,EACA;AACF,GAMW;AACT,QAAM,MAAM,IAAI,IAAI,CAAC,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAEnD,MAAI,IAAI,aAAa,UAAU;AAC7B,QAAI,WAAW;AAAA,EACjB;AAGA,MAAI,CAAC,IAAI,YAAY,CAAC,IAAI,OAAO,SAAS,EAAE,SAAS,IAAI,SAAS,QAAQ,OAAO,EAAE,CAAC,GAAG;AACrF,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE,IAAI;AAAA,EACnD,OAAO;AACL,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE;AAAA,EAC/C;AAEA,QAAM,cAAsC,CAAC;AAC7C,MAAI,SAAS;AACX,QAAI,YAAY;AACd,kBAAY,aAAa,IAAI;AAAA,IAC/B;AACA,QAAI,iBAAiB;AACnB,kBAAY,YAAY,IAAI;AAAA,IAC9B;AAAA,EACF,OAAO;AACL,gBAAY,OAAO,IAAI;AAAA,EACzB;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,QAAI,aAAa,IAAI,KAAK,KAAK;AAAA,EACjC;AAEA,SAAO,IAAI,SAAS;AACtB;AAYO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,gBAAuC,IAAI,IAAI,kBAAkB;AAAA,EACjE,iBAAiB,IAAI,MAA6B;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAiE,CAAC;AAAA,EAElE,0BAAkC;AAAA,EAElC,oBAA8C,CAAC;AAAA,EAC/C,oBAA8C,CAAC;AAAA,EAE/C,oBAAoB,IAAI,MAAM;AAAA,EAC9B,oBAAoB,IAAI,MAAM;AAAA;AAAA,EAG9B,UAAU,IAAI,gBAAgB,aAAa,cAAc,cAAc,EAAE;AAAA,EAEzE,mBAA2B;AAAA,EAEnC,UAAU,IAAI;AAAA,EACd;AAAA,EACA,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,mBAAmB;AAExB,SAAK,QAAQ,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,UAAU,MAAM,CAAC;AAE7D,SAAK,UAAU,KAAK,yBAAyB,CAAC;AAAA,EAChD;AAAA,EAEA,UAAU,SAAsC;AAC9C,SAAK,eAAe,IAAI,OAAO;AAAA,EACjC;AAAA,EAEQ,2BAAyD;AAG/D,UAAM,aAAyB,KAAK,iBAAiB,SAAS,WAAW,SAAS,OAAO,IACrF,CAAC,QAAQ,OAAO,IAChB,CAAC,MAAM;AAEX,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,oBAAoB;AAAA,QACpB,qBAAqB;AAAA,QACrB;AAAA,QACA,gBAAgB,KAAK,iBAAiB,SAAS;AAAA,QAC/C,2BAA2B,KAAK,iBAAiB,SAAS;AAAA;AAAA,QAE1D,aAAa,KAAK,iBAAiB,SAAS;AAAA,QAC5C,aAAa,gBAAgB,KAAK,iBAAiB,SAAS,UAAU;AAAA,QACtE,4BACE,KAAK,iBAAiB,SAAS,4BAA4B,WACvD,QACA,KAAK,iBAAiB,SAAS;AAAA;AAAA,QAErC,cAAc,KAAK;AAAA,QACnB,OAAO,KAAK,iBAAiB,SAAS;AAAA,MACxC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,UAAU;AACZ,WAAO,KAAK,cAAc,UAAU;AAAA,EACtC;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,UAA0C;AAC5D,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,SAAS,KAAK,0BAA0B,QAAQ;AACtD,UAAM,UAA0B,CAAC;AAEjC,eAAW,SAAS,QAAQ;AAC1B,YAAM,SAAS,IAAI,OAAa;AAChC,cAAQ,KAAK,MAAM;AAEnB,UAAI,MAAM,SAAS,4BAA4B;AAC7C,aAAK,kBAAkB,MAAM,KAAK,EAAE,IAAI;AAAA,MAC1C,WAAW,MAAM,QAAQ,4BAA4B;AACnD,aAAK,kBAAkB,MAAM,OAAO,IAAI;AAAA,MAC1C;AAEA,WAAK,UAAU,KAAK;AAAA,IACtB;AAEA,QAAI,QAAQ,WAAW,GAAG;AACxB,aAAO;AACP;AAAA,IACF;AAEA,QAAI;AAEF,YAAM,QAAQ,KAAK;AAAA,QACjB,QAAQ,IAAI,OAAO;AAAA,QACnB,MAAM,GAAI,EAAE,KAAK,MAAM;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACpD,CAAC;AAAA,MACH,CAAC;AAAA,IACH,SAAS,GAAG;AACV,WAAK,QAAQ,MAAO,EAAY,OAAO;AACvC,YAAM;AAAA,IACR,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEQ,0BACN,SACA,eAAwB,OAC2D;AACnF,UAAM,aAAa,QAAQ,KAAK;AAChC,QAAI,cAAc;AAChB,iBAAW,MAAM,KAAK,oBAAoB,CAAC;AAAA,IAC7C,OAAO;AAEL,iBAAW,QAAQ,WAAW,MAAM;AAAA,QAClC,CAAC,SAAS,CAAC,KAAK,GAAG,WAAW,oBAAoB;AAAA,MACpD;AAAA,IACF;AAEA,UAAM,SAGA,CAAC;AAEP,UAAM,UAAU,IAAI,mBAAmB,KAAK,SAAS,UAAU;AAC/D,eAAW,MAAM,QAAQ,UAAU;AACjC,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,SAAS;AAAA,QACT,UAAU,UAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AAEA,eAAW,CAAC,YAAY,EAAE,KAAK,QAAQ,UAAU;AAC/C,YAAM,WAAW,WAAW,QAAQ,EAAE;AACtC,UAAI,CAAC,UAAU;AACb,cAAM,IAAI,MAAM,aAAa,EAAE,YAAY;AAAA,MAC7C;AACA,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,MAAM,wBAAwB,QAAQ;AAAA,QACtC,kBAAkB,cAAc;AAAA,QAChC,UAAU,UAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,QAAwC;AACxD,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,KAAK,KAAK,uBAAuB,MAAM;AAC7C,SAAK,UAAU,EAAE;AAEjB,QAAI,CAAC,GAAG,QAAQ,OAAO;AACrB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAIA,UAAM,oBAAoB,IAAI,IAAI,GAAG,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC;AAC3E,UAAM,gBAAgB,OAAO;AAAA,MAC3B,OAAO,QAAQ,MAAM,EAAE;AAAA,QACrB,CAAC,CAAC,MAAM,IAAI,MAAM,IAAI,eAAe,IAAI,KAAK,kBAAkB,IAAI,IAAI;AAAA,MAC1E;AAAA,IACF;AAEA,SAAK,SAAS;AAEd,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB,QAAuD;AACpF,UAAM,WAA6B,CAAC;AAEpC,eAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,UAAI,CAAC,IAAI,eAAe,IAAI,GAAG;AAC7B,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAEA,YAAM,EAAE,YAAY,gBAAgB,YAAY,IAAI;AACpD,UAAI;AACF,cAAM,aAAa,IAAI;AAAA,UACrB;AAAA,QACF;AAEA,iBAAS,KAAK;AAAA,UACZ;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR,CAAC;AAAA,MACH,SAAS,GAAG;AACV,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,OAAO;AAAA,MACT;AAAA,MACA,UAAU,UAAU,eAAe;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,eAAsC;AAC7D,UAAM,UAAU,UAAU,sBAAsB;AAChD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,QACP,cAAc;AAAA,MAChB;AAAA,MACA,UAAU;AAAA,IACZ,CAAiC;AACjC,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,cAAc,EAAE,WAAW,GAA0C;AACnE,UAAM,UAAmD,CAAC;AAE1D,SAAK,iBAAiB,SAAS,aAAa;AAC5C,YAAQ,cAAc,gBAAgB,UAAU;AAIhD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,MACT,UAAU,UAAU,iBAAiB;AAAA,IACvC,CAAC;AAAA,EACH;AAAA,EAEA,UAAU,OAAyB;AACjC,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,aAAK,UAAU;AAAA,UACb,MAAM;AAAA,UACN,OAAO,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,QACtD,CAA0C;AAE1C,aAAK,oBAAqB,GAAG,oBAAoB,GAAG,aAAc;AAAA,MACpE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAA6B;AACjC,QAAI,KAAK,mBAAmB,KAAK;AAE/B,WAAK,UAAU;AAAA,QACb,MAAM;AAAA,MACR,CAA0C;AAC1C,WAAK,mBAAmB;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAyC;AACzC,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,UAAM,SAAS,KAAK,eAAe,EAAE,cAAc,eAAe,KAAK,CAAC;AACxE,SAAK,0BAA0B;AAC/B,WAAO,OAAO,QAAQ;AAAA,EACxB;AAAA,EAEA,MAAM,YAA2B;AAC/B,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAkC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,UAKG;AAChB,QAAI,CAAC,SAAS,cAAc,SAAS,WAAW,SAAS,OAAO,GAAG;AACjE,WAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,eAAe;AAAA,QACf,SAAS,SAAS;AAAA,QAClB,cAAc,SAAS;AAAA,MACzB,CAA4C;AAAA,IAC9C,WAAW,SAAS,oBAAoB,QAAW;AAEjD,YAAM,UAAU,KAAK,QAAQ,KAAK;AAClC,YAAM,MAAM,QAAQ,UAAU,SAAS,SAAS;AAChD,UAAI,QAAQ,QAAW;AACrB,cAAM,OAAO,QAAQ,MAAM,GAAG;AAC9B,YAAI,QAAQ,KAAK,SAAS,WAAW;AACnC,gBAAM,UAAU,IAAI,YAAY,OAAO;AAAA,YACrC,GAAG;AAAA,YACH,SAAS,CAAC,SAAS,eAAe;AAAA,UACpC,CAAC;AACD,kBAAQ,MAAM,GAAG,IAAI;AACrB,gBAAM,SAAS,KAAK,0BAA0B,OAAO;AACrD,qBAAW,MAAM,QAAQ;AACvB,iBAAK,UAAU,EAAE;AAAA,UACnB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,cACN,OACyB;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,aAAO,EAAE,GAAG,cAAc,OAAO,MAAM;AAAA,IACzC;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,aAAO,EAAE,GAAG,cAAc,OAAO,MAAM;AAAA,IACzC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,eAAmC;AAC/C,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,IAChB;AAEA,QAAI,KAAK,iBAAiB,SAAS,SAAS;AAI1C,UAAI,KAAK,iBAAiB,SAAS,YAAY;AAC7C,gBAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,UAAU;AAAA,MAC7E,WAAW,KAAK,iBAAiB,SAAS,QAAQ;AAChD,gBAAQ,SAAS,IAAI,KAAK,iBAAiB,SAAS;AAAA,MACtD,OAAO;AACL,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAAA,IACF,OAAO;AACL,cAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,MAAM;AACvE,cAAQ,aAAa,IAAI;AAAA,IAC3B;AAEA,UAAM,MAAM,eAAe;AAAA,MACzB,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,OAAO,KAAK,iBAAiB,SAAS;AAAA,MACtC,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,YAAY,KAAK,iBAAiB,SAAS;AAAA,MAC3C,iBAAiB,KAAK,iBAAiB,SAAS;AAAA,IAClD,CAAC;AAED,QAAI,YAAY;AACd,WAAK,QAAQ,MAAM,wCAAwC,GAAG,EAAE;AAAA,IAClE;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,KAAK,IAAI,UAAU,KAAK,EAAE,QAAQ,CAAC;AACzC,UAAI,UAAU;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,WAAG,MAAM;AACT,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,iBAAiB,SAAS,YAAY,SAAS;AAEvD,SAAG,KAAK,QAAQ,MAAM;AACpB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,gBAAQ,EAAE;AAAA,MACZ,CAAC;AAED,SAAG,KAAK,SAAS,MAAM;AACrB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,eAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,QAAoC;AAClD,QAAI,eAAe;AACnB,QAAI,aAAa;AACjB,QAAI,SAA2B;AAC/B,UAAM,aAAa,KAAK,iBAAiB,SAAS,YAAY;AAE9D,UAAM,YAAY,YAAY;AAC5B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,oBAAoB,KAAK,iBAAiB,SAAS;AAAA,QACrD;AAAA,QACA;AAAA,MACF;AAEA,YAAM,SAAkC,CAAC;AAGzC,aAAO,KAAK,KAAK,yBAAyB,CAAC;AAG3C,UAAI,OAAO,KAAK,KAAK,MAAM,EAAE,SAAS,GAAG;AACvC,eAAO,KAAK,KAAK,uBAAuB,KAAK,MAAM,CAAC;AAAA,MACtD;AAGA,YAAM,UAAU,KAAK,QAAQ,KAAK;AAAA,QAChC,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,MACvB,CAAC;AAED,YAAM,aAAa,KAAK;AACxB,WAAK,gBAAgB,IAAI,IAAI,kBAAkB;AAC/C,aAAO,KAAK,GAAG,KAAK,0BAA0B,OAAO,CAAC;AAEtD,UAAI;AACF,mBAAW,MAAM,QAAQ;AACvB,eAAK,KAAK,8BAA8B,EAAE;AAC1C,iBAAQ,KAAK,KAAK,UAAU,EAAE,CAAC;AAAA,QACjC;AAAA,MACF,SAAS,OAAO;AACd,aAAK,gBAAgB;AACrB,cAAM,IAAI,mBAAmB;AAAA,UAC3B,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ,MAAM,oCAAoC;AAEvD,WAAK,KAAK,uBAAuB,CAAC,CAAwC;AAAA,IAC5E;AAEA,mBAAe;AACf,WAAO,CAAC,KAAK,WAAW,CAAC,OAAO,SAAS;AACvC,WAAK,QAAQ,MAAM,sDAAsD;AACzE,eAAS,MAAM,KAAK,aAAa;AACjC,UAAI,OAAO,QAAS;AAEpB,UAAI;AACF,YAAI,cAAc;AAChB,gBAAM,UAAU;AAChB,cAAI,OAAO,QAAS;AACpB,uBAAa;AAAA,QACf;AAEA,cAAM,KAAK,MAAM,MAAM;AACvB,YAAI,OAAO,QAAS;AAAA,MACtB,SAAS,OAAO;AACd,YAAI,CAAC,WAAW,KAAK,GAAG;AACtB,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,KAAK,CAAC,MAAM,WAAW;AACxC,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,YAAY;AAC7B,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,+CAA+C,UAAU;AAAA,YAClE,SAAS;AAAA,cACP,MAAM;AAAA,cACN,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAEA,aAAK,UAAU,EAAE,OAAuB,aAAa,KAAK,CAAC;AAC3D,cAAM,gBACJ,eAAe,IACX,kCACA,KAAK,iBAAiB,SAAS,YAAY;AACjD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS;AAAA,YACT;AAAA,YACA;AAAA,UACF;AAAA,UACA,sDAAsD,gBAAgB,GAAI;AAAA,QAC5E;AAEA,cAAM,MAAM,aAAa;AACzB;AAAA,MACF;AAEA,qBAAe;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAc,MAAM,QAAkC;AACpD,UAAM,gBAAgB,OAAO,WAAuC;AAClE,YAAM,cAAc,IAAI,OAAa;AACrC,aAAO,iBAAiB,SAAS,MAAM,YAAY,QAAQ,CAAC;AAE5D,aAAO,CAAC,KAAK,WAAW,OAAO,eAAe,UAAU,QAAQ,CAAC,OAAO,SAAS;AAC/E,YAAI;AACF,gBAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,eAAe,IAAI,GAAG,YAAY,KAAK,CAAC;AAC/E,cAAI,OAAO,WAAW,YAAY,QAAQ,UAAU,QAAW;AAC7D;AAAA,UACF;AAEA,cAAI,YAAY;AACd,iBAAK,QAAQ,MAAM,KAAK,cAAc,KAAK,GAAG,eAAe,MAAM,IAAI,EAAE;AAAA,UAC3E;AAEA,eAAK,KAAK,8BAA8B,KAAK;AAC7C,iBAAO,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,QACnC,SAAS,OAAO;AACd;AAAA,QACF;AAAA,MACF;AAEA,aAAO,MAAM;AAAA,IACf;AAEA,UAAM,gBAAgB,IAAI,OAAqB;AAE/C,WAAO,UAAU,CAAC,UAAU;AAC1B,oBAAc,QAAQ,IAAI,mBAAmB,EAAE,SAAS,MAAM,QAAQ,CAAC,CAAC;AAAA,IAC1E;AACA,WAAO,UAAU,MAAM;AACrB,oBAAc,QAAQ;AAAA,IACxB;AAEA,WAAO,YAAY,CAAC,YAA0B;AAC5C,YAAM,QAA+B,KAAK,MAAM,QAAQ,IAAc;AAEtE,WAAK,KAAK,gCAAgC,KAAK;AAC/C,UAAI,YAAY;AACd,aAAK,QAAQ,MAAM,KAAK,cAAc,KAAK,GAAG,eAAe,MAAM,IAAI,EAAE;AAAA,MAC3E;AAEA,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,sBAAsB,KAAK;AAChC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,uDAAuD,KAAK;AACjE;AAAA,QACF,KAAK;AACH,eAAK,oDAAoD,KAAK;AAC9D;AAAA,QACF,KAAK;AACH,eAAK,+BAA+B,KAAK;AACzC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,wBAAwB,KAAK;AAClC;AAAA,QACF,KAAK;AACH,eAAK,uBAAuB,KAAK;AACjC;AAAA,QACF,KAAK;AACH,eAAK,mCAAmC,KAAK;AAC7C;AAAA,QACF,KAAK;AACH,eAAK,yBAAyB,KAAK;AACnC;AAAA,QACF,KAAK;AACH,eAAK,kCAAkC,KAAK;AAC5C;AAAA,QACF,KAAK;AACH,eAAK,wBAAwB,KAAK;AAClC;AAAA,QACF,KAAK;AACH,eAAK,6BAA6B,KAAK;AACvC;AAAA,QACF,KAAK;AACH,eAAK,mBAAmB,KAAK;AAC7B;AAAA,QACF,KAAK;AACH,eAAK,YAAY,KAAK;AACtB;AAAA,QACF;AACE,cAAI,YAAY;AACd,iBAAK,QAAQ,MAAM,oBAAoB,MAAM,IAAI,EAAE;AAAA,UACrD;AACA;AAAA,MACJ;AAAA,IACF;AAEA,UAAM,WAAW,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,cAAc,MAAM,CAAC;AAEhE,UAAM,SAAS,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AACvC,YAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,eAAO,iBAAiB,SAAS,MAAM;AACrC,kBAAQ;AAAA,QACV,CAAC;AAAA,MACH,CAAC;AAED,aAAO,QAAQ,KAAK,CAAC,cAAc,OAAO,YAAY,CAAC;AAAA,IACzD,CAAC;AAED,UAAM,oBAAoB,KAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AACxD,YAAM,MAAM,KAAK,iBAAiB,SAAS,oBAAoB,EAAE,OAAO,CAAC;AACzE,aAAO,IAAI,mBAAmB;AAAA,QAC5B,SAAS;AAAA,MACX,CAAC;AAAA,IACH,CAAC;AAED,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,OAAO,QAAQ,SAAS,QAAQ,kBAAkB,MAAM,CAAC;AAE5F,UAAI,kBAAkB,QAAQ,KAAK,mBAAmB;AACpD,cAAM,KAAK,kBAAkB,SAAS;AAAA,MACxC;AAEA,UAAI,kBAAkB,OAAO;AAC3B,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,YAAM,cAAc,CAAC,QAAQ,UAAU,iBAAiB,GAAG,GAAI;AAC/D,aAAO,MAAM;AAAA,IACf;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,MAAM;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAAA,EACb;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B,KAAK,iBAAiB,SAAS,4BAA4B;AAAA,IACvF,CAAgC;AAAA,EAClC;AAAA,EAEQ,sBAAsB,OAA6C;AAthC7E;AAuhCI,QAAI,CAAC,MAAM,SAAS,IAAI;AACtB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D,UAAU,oBAAI,IAAI;AAAA,MAClB,UAAU,IAAI,OAAO;AAAA,MACrB,mBAAmB,KAAK,IAAI;AAAA,IAC9B;AAIA,UAAM,eAAe;AAAA,MACnB,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,MACf,YAAY,MAAM,SAAS;AAAA,IAC7B;AAEA,UAAM,iBAAgB,WAAM,SAAS,aAAf,mBAAyB;AAC/C,QAAI,eAAe;AACjB,YAAM,SAAS,KAAK,uBAAuB,aAAa;AACxD,UAAI,QAAQ;AACV,eAAO,KAAK,uBAAuB,aAAa;AAChD,qBAAa,gBAAgB;AAC7B,YAAI,CAAC,OAAO,QAAQ,MAAM;AACxB,iBAAO,QAAQ,QAAQ,YAAY;AAAA,QACrC;AAAA,MACF;AAAA,IACF;AAEA,SAAK,KAAK,sBAAsB,YAAY;AAAA,EAC9C;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,QAAI,CAAC,MAAM,KAAK,MAAM;AACpB,YAAM,IAAI,MAAM,sBAAsB;AAAA,IACxC;AAEA,QAAI,CAAC,MAAM,aAAa;AACtB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,UAAM,WAAW,MAAM,KAAK;AAC5B,UAAM,aAAa,MAAM;AAEzB,QAAI,aAAa,WAAW;AAE1B,WAAK,kBAAkB,UAAU;AACjC,WAAK,0BAA0B;AAC/B;AAAA,IACF;AAEA,UAAM,SAAS,MAAM,KAAK;AAC1B,QAAI,CAAC,QAAQ;AACX,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,gBAAgB,IAAI,OAAmB;AAC7C,UAAM,iBAAoC;AAAA,MACxC,WAAW;AAAA,MACX,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,iBAAiB;AAAA,MACjB,YAAY;AAAA,IACd;AAGA,QAAI,CAAC,KAAK,iBAAiB,aAAa,aAAa;AACnD,qBAAe,aAAa,MAAM;AAClC,oBAAc,QAAQ,CAAC,MAAM,CAAC;AAAA,IAChC;AAEA,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,eAAe,YAAY,OAAO;AAAA,MAC9C,aAAa,eAAe,aAAa,OAAO;AAAA,MAChD,YAAY,cAAc;AAAA,IAC5B,CAAC;AAED,SAAK,kBAAkB,SAAS,IAAI,QAAQ,cAAc;AAAA,EAC5D;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,KAAK,IAAI;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,kBAAkB,wBAAwB,MAAM,IAAI,CAAC;AAAA,IACvF,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,KAAK,GAAG,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,OAAO;AAAA,IACzC,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,QAAQ,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,OAAO;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,OAAO;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,uDACN,OACM;AACN,UAAM,aAAa,KAAK,cAAc,IAAI,MAAM,OAAO;AACvD,QAAI,CAAC,YAAY;AACf;AAAA,IACF;AAEA,UAAM,OAAO,WAAW;AACxB,QAAI,gBAAgB,IAAI,aAAa;AACnC,WAAK,QAAQ,KAAK,MAAM,UAAU;AAAA,IACpC,OAAO;AACL,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,SAAK,KAAK,uCAAuC;AAAA,MAC/C,QAAQ,MAAM;AAAA,MACd,YAAY,MAAM;AAAA,MAClB,SAAS;AAAA,IACX,CAAoC;AAAA,EACtC;AAAA,EAEQ,oDACN,OACM;AACN,SAAK,QAAQ;AAAA,MACX,EAAE,OAAO,MAAM,MAAM;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,+BAA+B,OAAsD;AAC3F,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AACrB,UAAM,WAAW,MAAM,KAAK;AAE5B,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,QAAI,CAAC,gBAAgB;AACnB,WAAK,QAAQ,KAAK,uCAAuC,MAAM,EAAE;AACjE;AAAA,IACF;AAEA,QAAI,aAAa,UAAU,KAAK,iBAAiB,aAAa,aAAa;AACzE,WAAK,QAAQ,KAAK,oEAAoE;AAAA,IACxF;AAEA,QAAI,CAAC,eAAe,WAAW,MAAM;AACnC,YAAM,iBAA6B,aAAa,SAAS,CAAC,MAAM,IAAI,CAAC,SAAS,MAAM;AACpF,qBAAe,WAAW,QAAQ,cAAc;AAAA,IAClD;AAEA,QAAI,KAAK,kBAAkB,yBAAyB,QAAW;AAC7D,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,MAAM;AACf;AAAA,IACF;AACA,QAAI,MAAM,KAAK,SAAS,QAAQ;AAC9B;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EAGF;AAAA,EAEQ,wBAAwB,OAA+C;AAC7E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM,OAAO;AACxE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,QACE,CAAC,KAAK,iBAAiB,aAAa,eACpC,CAAC,KAAK,kBAAkB,sBACxB;AACA,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAEA,mBAAe,YAAY,MAAM,MAAM,KAAK;AAC5C,mBAAe,mBAAmB,MAAM;AAAA,EAC1C;AAAA,EAEQ,uBAAuB,QAA+C;AAC5E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,mCACN,OACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AACrB,UAAM,QAAQ,MAAM;AAIpB,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C,OAAO;AACL,qBAAe,YAAY,MAAM,KAAK;AACtC,qBAAe,mBAAmB;AAAA,IACpC;AAAA,EACF;AAAA,EAEQ,yBAAyB,OAAgD;AAC/E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM,OAAO;AACxE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,QAAI,KAAK,kBAAkB,yBAAyB,QAAW;AAC7D,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAEA,QAAI,CAAC,eAAe,WAAW,MAAM;AACnC,qBAAe,WAAW,QAAQ,CAAC,SAAS,MAAM,CAAC;AAAA,IACrD;AAEA,UAAM,eAAe,KAAK,MAAM,KAAK;AACrC,UAAM,MAAM,aAAa;AACzB,UAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,aAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,YAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,IACtC;AAEA,mBAAe,aAAa;AAAA,MAC1B,IAAI;AAAA,QACF,IAAI,WAAW,MAAM,MAAM;AAAA,QAC3B,UAAU;AAAA,QACV,UAAU;AAAA,QACV,MAAM,SAAS;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kCACN,QACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,wBAAwB,QAAgD;AAC9E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,6BAA6B,OAAoD;AACvF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM,KAAK;AAC1B,UAAM,WAAW,MAAM,KAAK;AAE5B,QAAI,aAAa,iBAAiB;AAChC,YAAM,OAAO,MAAM;AACnB,UAAI,CAAC,KAAK,WAAW,CAAC,KAAK,QAAQ,CAAC,KAAK,WAAW;AAClD,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,WAAK,kBAAkB,gBAAgB;AAAA,QACrC,IAAI,aAAa,OAAO;AAAA,UACtB,QAAQ,KAAK;AAAA,UACb,MAAM,KAAK;AAAA,UACX,MAAM,KAAK;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF,WAAW,aAAa,WAAW;AACjC,YAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,UAAI,CAAC,gBAAgB;AACnB;AAAA,MACF;AAEA,qBAAe,YAAY,MAAM;AACjC,qBAAe,aAAa,MAAM;AAClC,UAAI,CAAC,eAAe,WAAW,MAAM;AAEnC,uBAAe,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,MAC7E;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,mBAAmB,QAA2C;AAl2CxE;AAm2CI,QAAI,CAAC,KAAK,mBAAmB;AAG3B;AAAA,IACF;AAEA,UAAM,mBAAmB,KAAK,kBAAkB;AAChD,UAAM,sBAAsB,KAAK,kBAAkB;AAEnD,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,cAAc,KAAK,kBAAkB,SAAS;AAAA,MAChD;AAAA,MACA;AAAA,IACF;AAEA,eAAW,cAAc,KAAK,kBAAkB,SAAS,OAAO,GAAG;AACjE,iBAAW,YAAY,MAAM;AAC7B,iBAAW,aAAa,MAAM;AAC9B,UAAI,CAAC,WAAW,WAAW,MAAM;AAC/B,mBAAW,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,MACzE;AAAA,IACF;AAEA,SAAK,kBAAkB,gBAAgB,MAAM;AAC7C,SAAK,kBAAkB,eAAe,MAAM;AAE5C,eAAW,UAAU,KAAK,kBAAkB,SAAS,KAAK,GAAG;AAC3D,YAAM,aAAa,KAAK,cAAc,IAAI,MAAM;AAChD,UAAI,cAAc,WAAW,gBAAgB,IAAI,aAAa;AAC5D,mBAAW,KAAK,QAAQ,KAAK,KAAK,kBAAkB,SAAS,IAAI,MAAM,EAAG,eAAe;AAAA,MAC3F;AAAA,IACF;AAEA,SAAK,kBAAkB,SAAS,QAAQ;AACxC,SAAK,oBAAoB;AAGzB,UAAM,QAAQ,OAAO,SAAS;AAC9B,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,KAAK,IAAI,IAAI;AAEhC,UAAM,kBAAgD;AAAA,MACpD,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,OAAO,SAAS,MAAM;AAAA,MACjC;AAAA,MACA;AAAA,MACA,WAAW,OAAO,SAAS,WAAW;AAAA,MACtC,OAAO;AAAA,MACP,cAAa,+BAAO,iBAAgB;AAAA,MACpC,eAAc,+BAAO,kBAAiB;AAAA,MACtC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,iBAAiB,aAAa,MAAK,+BAAO,kBAAiB,MAAM,aAAa,OAAQ;AAAA,MACtF,mBAAmB;AAAA,QACjB,eAAa,oCAAO,wBAAP,mBAA4B,iBAAgB;AAAA,QACzD,cAAY,oCAAO,wBAAP,mBAA4B,gBAAe;AAAA,QACvD,aAAa;AAAA;AAAA,QACb,gBAAc,oCAAO,wBAAP,mBAA4B,kBAAiB;AAAA,QAC3D,uBAAqB,oCAAO,wBAAP,mBAA4B,yBAC7C;AAAA,UACE,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,UAChF,cAAY,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,gBAAe;AAAA,UAC9E,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,QAClF,IACA;AAAA,MACN;AAAA,MACA,oBAAoB;AAAA,QAClB,cAAY,oCAAO,yBAAP,mBAA6B,gBAAe;AAAA,QACxD,eAAa,oCAAO,yBAAP,mBAA6B,iBAAgB;AAAA,QAC1D,aAAa;AAAA,MACf;AAAA,IACF;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAEhD;AAAA,EAEQ,YAAY,OAAmC;AACrD,QAAI,MAAM,MAAM,QAAQ,WAAW,qBAAqB,GAAG;AACzD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,OAAO,MAAM,MAAM,GAAG,uCAAuC;AAClF,SAAK,UAAU;AAAA,MACb,OAAO,IAAI,SAAS,MAAM,MAAM,SAAS;AAAA,QACvC,MAAM,MAAM;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAAA,MACD,aAAa;AAAA,IACf,CAAC;AAAA,EAGH;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAAiD;AAEtF,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAA2B;AAAA,EAC7B;AAAA,EAEA,CAAS,cAAc,OAA0C;AAC/D,UAAM;AAAA,EACR;AAAA,EAEQ,eAAe;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIyB;AACvB,UAAM,SAAS,aAAa,IAAI,qBAAqB,EAAE,aAAa,CAAC;AACrE,QAAI,aAAa,cAAc;AAC7B,aAAO,eAAe;AAAA,IACxB;AAEA,UAAM,UAAU,UAAU,kBAAkB;AAC5C,QAAI,eAAe;AACjB,WAAK,uBAAuB,OAAO,IAAI;AAAA,IACzC;AAEA,UAAM,WAAsD,CAAC;AAC7D,QAAI,aAAc,UAAS,eAAe;AAC1C,QAAI,cAAe,UAAS,WAAW,EAAE,iBAAiB,QAAQ;AAElE,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,UAAU;AAAA,MACV,UAAU,OAAO,KAAK,QAAQ,EAAE,SAAS,IAAI,WAAW;AAAA,IAC1D,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEQ,kBAAkB,YAA0B;AAClD,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,gBAAgB;AAAA,MACpB,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,MACf;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,uBAAuB,UAAU;AACrD,QAAI,QAAQ;AACV,aAAO,KAAK,uBAAuB,UAAU;AAC7C,oBAAc,gBAAgB;AAC9B,UAAI,OAAO,QAAQ,MAAM;AACvB,aAAK,QAAQ,KAAK,EAAE,WAAW,GAAG,iCAAiC;AAAA,MACrE,OAAO;AACL,eAAO,QAAQ,QAAQ,aAAa;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,wBAAwB,MAA4C;AAC3E,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,MAAM,KAAK;AAAA,QACX,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,QAAQ,KAAK;AAAA,MACf;AAAA,IACF,KAAK;AACH,YAAM,OAAO,KAAK,SAAS,cAAc,WAAW,KAAK;AACzD,YAAM,cAAmC,CAAC;AAC1C,iBAAW,KAAK,KAAK,SAAS;AAC5B,YAAI,OAAO,MAAM,UAAU;AACzB,sBAAY,KAAK;AAAA,YACf,MAAM,SAAS,cAAc,SAAS;AAAA,YACtC,MAAM;AAAA,UACR,CAA+B;AAAA,QACjC,WAAW,EAAE,SAAS,iBAAiB;AAErC;AAAA,QACF,WAAW,EAAE,SAAS,iBAAiB;AACrC,cAAI,SAAS,QAAQ;AACnB,kBAAM,eAAe,OAAO,KAAK,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,QAAQ;AACpF,wBAAY,KAAK;AAAA,cACf,MAAM;AAAA,cACN,OAAO;AAAA,YACT,CAAgC;AAAA,UAClC;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN;AAAA,QACA,SAAS;AAAA,MACX;AAAA,IACF;AACE,YAAM,IAAI,MAAM,0BAA2B,KAAa,IAAI,EAAE;AAAA,EAClE;AACF;AAEA,SAAS,wBAAwB,MAA4C;AAC3E,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,oBAAoB;AAAA,EACtC;AAEA,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO,IAAI,aAAa,OAAO;AAAA,QAC7B,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,MAAM,KAAK;AAAA,QACX,MAAM,KAAK;AAAA,MACb,CAAC;AAAA,IACH,KAAK;AACH,aAAO,IAAI,mBAAmB,OAAO;AAAA,QACnC,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,QAAQ,KAAK;AAAA,QACb,SAAS;AAAA,MACX,CAAC;AAAA,IACH,KAAK;AACH,YAAM,UAA6B,CAAC;AAEpC,YAAM,WAAW,MAAM,QAAQ,KAAK,OAAO,IAAI,KAAK,UAAU,CAAC,KAAK,OAAO;AAC3E,iBAAW,KAAK,UAAU;AACxB,YAAI,EAAE,SAAS,UAAU,EAAE,SAAS,cAAc;AAChD,kBAAQ,KAAK,EAAE,IAAI;AAAA,QACrB;AAAA,MACF;AACA,aAAO,IAAI,YAAY,OAAO;AAAA,QAC5B,IAAI,KAAK;AAAA,QACT,MAAM,KAAK;AAAA,QACX;AAAA,MACF,CAAC;AAAA,EACL;AACF;AAEA,SAAS,oBAAoB,kBAA0B,GAAoB;AACzE,QAAM,YAAY,OAAO,MAAM,kBAAkB,WAAW;AAC5D,SAAO,IAAI,YAAY,OAAO;AAAA,IAC5B,IAAI,UAAU,oBAAoB;AAAA,IAClC,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,QACE,MAAM;AAAA,QACN,OAAO;AAAA,UACL,IAAI;AAAA,YACF,IAAI,WAAW,UAAU,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,YACA,UAAU,SAAS;AAAA,UACrB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,gBAAgB,YAAmD;AAC1E,MAAI,OAAO,eAAe,UAAU;AAClC,WAAO;AAAA,EACT;AAEA,OAAI,yCAAY,UAAS,YAAY;AACnC,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,SAAO;AACT;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-openai",
-  "version": "1.0.30",
+  "version": "1.0.32",
   "description": "OpenAI plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
@@ -30,9 +30,9 @@
     "@types/ws": "^8.5.10",
     "tsup": "^8.3.5",
     "typescript": "^5.0.0",
-    "@livekit/agents-plugin-silero": "1.0.30",
-    "@livekit/agents-plugins-test": "1.0.30",
-    "@livekit/agents": "1.0.30"
+    "@livekit/agents": "1.0.32",
+    "@livekit/agents-plugin-silero": "1.0.32",
+    "@livekit/agents-plugins-test": "1.0.32"
   },
   "dependencies": {
     "@livekit/mutex": "^1.1.1",
@@ -41,7 +41,7 @@
   },
   "peerDependencies": {
     "@livekit/rtc-node": "^0.13.22",
-    "@livekit/agents": "1.0.30"
+    "@livekit/agents": "1.0.32"
   },
   "scripts": {
     "build": "tsup --onSuccess \"pnpm build:types\"",

package/src/realtime/api_proto.ts

@@ -20,7 +20,9 @@ export type Voice =
   | 'sage'
   | 'verse'
   | string;
-export type AudioFormat = 'pcm16'; // TODO: 'g711-ulaw' | 'g711-alaw'
+
+/** @deprecated Use LegacyAudioFormat for Beta API string format or AudioFormat for GA API object format */
+export type LegacyAudioFormat = 'pcm16'; // TODO: 'g711-ulaw' | 'g711-alaw' (Beta format)
 export type Role = 'system' | 'assistant' | 'user' | 'tool';
 export type GenerationFinishedReason = 'stop' | 'max_tokens' | 'content_filter' | 'interrupt';
 export type InputTranscriptionModel = 'whisper-1' | string; // Open-ended, for future models
@@ -44,6 +46,7 @@ export type ClientEventType =
   | 'conversation.item.delete'
   | 'response.create'
   | 'response.cancel';
+
 export type ServerEventType =
   | 'error'
   | 'session.created'
@@ -53,7 +56,8 @@ export type ServerEventType =
   | 'input_audio_buffer.cleared'
   | 'input_audio_buffer.speech_started'
   | 'input_audio_buffer.speech_stopped'
-  | 'conversation.item.created'
+  | 'conversation.item.added' // GA: renamed from conversation.item.created
+  | 'conversation.item.created' // Beta: kept for backward compatibility
   | 'conversation.item.input_audio_transcription.completed'
   | 'conversation.item.input_audio_transcription.failed'
   | 'conversation.item.truncated'
@@ -64,12 +68,18 @@ export type ServerEventType =
   | 'response.output_item.done'
   | 'response.content_part.added'
   | 'response.content_part.done'
-  | 'response.text.delta'
-  | 'response.text.done'
-  | 'response.audio_transcript.delta'
-  | 'response.audio_transcript.done'
-  | 'response.audio.delta'
-  | 'response.audio.done'
+  | 'response.output_text.delta' // GA: renamed from response.text.delta
+  | 'response.output_text.done' // GA: renamed from response.text.done
+  | 'response.text.delta' // Beta: kept for backward compatibility
+  | 'response.text.done' // Beta: kept for backward compatibility
+  | 'response.output_audio_transcript.delta' // GA: renamed from response.audio_transcript.delta
+  | 'response.output_audio_transcript.done' // GA: renamed from response.audio_transcript.done
+  | 'response.audio_transcript.delta' // Beta: kept for backward compatibility
+  | 'response.audio_transcript.done' // Beta: kept for backward compatibility
+  | 'response.output_audio.delta' // GA: renamed from response.audio.delta
+  | 'response.output_audio.done' // GA: renamed from response.audio.done
+  | 'response.audio.delta' // Beta: kept for backward compatibility
+  | 'response.audio.done' // Beta: kept for backward compatibility
   | 'response.function_call_arguments.delta'
   | 'response.function_call_arguments.done'
   | 'rate_limits.updated';
@@ -113,6 +123,35 @@ export type InputAudioTranscription = {
   prompt?: string;
 };
 
+export type NoiseReductionType = 'near_field' | 'far_field';
+
+export interface NoiseReduction {
+  type: NoiseReductionType;
+}
+
+export interface AudioFormat {
+  type: 'audio/pcm';
+  rate: number;
+}
+
+export interface RealtimeAudioConfigInput {
+  format?: AudioFormat;
+  noise_reduction?: NoiseReduction | null;
+  transcription?: InputAudioTranscription | null;
+  turn_detection?: TurnDetectionType | null;
+}
+
+export interface RealtimeAudioConfigOutput {
+  format?: AudioFormat;
+  speed?: number;
+  voice?: Voice;
+}
+
+export interface RealtimeAudioConfig {
+  input?: RealtimeAudioConfigInput;
+  output?: RealtimeAudioConfigOutput;
+}
+
 export interface InputTextContent {
   type: 'input_text';
   text: string;
@@ -136,9 +175,10 @@ export interface AudioContent {
 
 export type Content = InputTextContent | InputAudioContent | TextContent | AudioContent;
 export type ContentPart = {
-  type: 'text' | 'audio';
+  type: 'text' | 'audio' | 'output_text' | 'output_audio'; // GA: output_text/output_audio
   audio?: AudioBase64Bytes;
   transcript?: string;
+  text?: string; // GA: text field for output_text
 };
 
 export interface BaseItem {
@@ -193,8 +233,8 @@ export interface SessionResource {
   modalities: Modality[]; // default: ["text", "audio"]
   instructions: string;
   voice: Voice; // default: "alloy"
-  input_audio_format: AudioFormat; // default: "pcm16"
-  output_audio_format: AudioFormat; // default: "pcm16"
+  input_audio_format: LegacyAudioFormat; // default: "pcm16"
+  output_audio_format: LegacyAudioFormat; // default: "pcm16"
   input_audio_transcription: InputAudioTranscription | null;
   turn_detection: TurnDetectionType | null;
   tools: Tool[];
@@ -266,22 +306,34 @@ interface BaseClientEvent {
 export interface SessionUpdateEvent extends BaseClientEvent {
   type: 'session.update';
   session: Partial<{
+    // GA fields
+    type?: 'realtime'; // GA: session type
+    output_modalities?: Modality[]; // GA: renamed from modalities
+    audio?: RealtimeAudioConfig; // GA: nested audio config
+    max_output_tokens?: number | 'inf'; // GA: renamed from max_response_output_tokens
+    tracing?: TracingConfig | null; // GA: tracing config
+    // Common fields
     model: Model;
-    modalities: Modality[];
     instructions: string;
+    tools: Tool[];
+    tool_choice: ToolChoice;
+    // Beta fields (kept for backward compatibility)
+    modalities: Modality[];
     voice: Voice;
-    input_audio_format: AudioFormat;
-    output_audio_format: AudioFormat;
+    input_audio_format: LegacyAudioFormat;
+    output_audio_format: LegacyAudioFormat;
     input_audio_transcription: InputAudioTranscription | null;
     turn_detection: TurnDetectionType | null;
-    tools: Tool[];
-    tool_choice: ToolChoice;
     temperature: number;
     max_response_output_tokens?: number | 'inf';
     speed?: number;
   }>;
 }
 
+export interface TracingConfig {
+  enabled?: boolean;
+}
+
 export interface InputAudioBufferAppendEvent extends BaseClientEvent {
   type: 'input_audio_buffer.append';
   audio: AudioBase64Bytes;
@@ -353,7 +405,7 @@ export interface ResponseCreateEvent extends BaseClientEvent {
     modalities: Modality[];
     instructions: string;
     voice: Voice;
-    output_audio_format: AudioFormat;
+    output_audio_format: LegacyAudioFormat;
     tools?: Tool[];
     tool_choice: ToolChoice;
     temperature: number;
@@ -435,6 +487,12 @@ export interface ConversationItemCreatedEvent extends BaseServerEvent {
   item: ItemResource;
 }
 
+export interface ConversationItemAddedEvent extends BaseServerEvent {
+  type: 'conversation.item.added';
+  previous_item_id: string;
+  item: ItemResource;
+}
+
 export interface ConversationItemInputAudioTranscriptionCompletedEvent extends BaseServerEvent {
   type: 'conversation.item.input_audio_transcription.completed';
   item_id: string;
@@ -593,6 +651,7 @@ export type ServerEvent =
   | InputAudioBufferSpeechStartedEvent
   | InputAudioBufferSpeechStoppedEvent
   | ConversationItemCreatedEvent
+  | ConversationItemAddedEvent // GA: renamed from conversation.item.created
   | ConversationItemInputAudioTranscriptionCompletedEvent
   | ConversationItemInputAudioTranscriptionFailedEvent
   | ConversationItemTruncatedEvent

package/src/realtime/index.ts

@@ -3,3 +3,4 @@
 // SPDX-License-Identifier: Apache-2.0
 export * from './api_proto.js';
 export * from './realtime_model.js';
+export * as beta from './realtime_model_beta.js';