@livekit/agents-plugin-openai 0.8.1 → 0.9.0

package/dist/llm.cjs

@@ -443,9 +443,7 @@ const buildMessage = async (msg, cacheKey) => {
   }
   if (typeof msg.content === "string") {
     oaiMsg.content = msg.content;
-  } else if (((c) => {
-    return c.length !== void 0;
-  })(msg.content)) {
+  } else if (Array.isArray(msg.content)) {
     oaiMsg.content = await Promise.all(
       msg.content.map(async (c) => {
         if (typeof c === "string") {
@@ -462,6 +460,8 @@ const buildMessage = async (msg, cacheKey) => {
         }
       })
     );
+  } else if (msg.content === void 0) {
+    oaiMsg.content = "";
   }
   if (msg.toolCalls && oaiMsg.role === "assistant") {
     oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({

package/dist/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this,\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n  label = 'openai.LLMStream';\n\n  constructor(\n    llm: LLM,\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(llm, chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (\n    ((c?: llm.ChatContent | llm.ChatContent[]): c is llm.ChatContent[] => {\n      return (c as llm.ChatContent[]).length !== undefined;\n    })(msg.content)\n  ) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAyB;AACzB,yBAA2B;AAC3B,oBAAoC;AACpC,mBAAkB;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,qBAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,0BAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,kBAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAM,+BAAW;AAAA,EACjB,QAAQ;AAAA,EAER,YACEA,MACA,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAMA,MAAK,SAAS,MAAM;AAC1B,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,kBAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,kBAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,kBAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,kBAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,YACG,CAAC,MAAoE;AACpE,WAAQ,EAAwB,WAAW;AAAA,EAC7C,GAAG,IAAI,OAAO,GACd;AACA,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACC,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,cAAU,aAAAC,SAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["llm","c","sharp"]}
+{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this,\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n  label = 'openai.LLMStream';\n\n  constructor(\n    llm: LLM,\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(llm, chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (Array.isArray(msg.content)) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  } else if (msg.content === undefined) {\n    oaiMsg.content = '';\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAyB;AACzB,yBAA2B;AAC3B,oBAAoC;AACpC,mBAAkB;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,qBAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,0BAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,kBAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAM,+BAAW;AAAA,EACjB,QAAQ;AAAA,EAER,YACEA,MACA,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAMA,MAAK,SAAS,MAAM;AAC1B,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,kBAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,kBAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,kBAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,kBAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACC,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF,WAAW,IAAI,YAAY,QAAW;AACpC,WAAO,UAAU;AAAA,EACnB;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,cAAU,aAAAC,SAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["llm","c","sharp"]}

package/dist/llm.js

@@ -409,9 +409,7 @@ const buildMessage = async (msg, cacheKey) => {
   }
   if (typeof msg.content === "string") {
     oaiMsg.content = msg.content;
-  } else if (((c) => {
-    return c.length !== void 0;
-  })(msg.content)) {
+  } else if (Array.isArray(msg.content)) {
     oaiMsg.content = await Promise.all(
       msg.content.map(async (c) => {
         if (typeof c === "string") {
@@ -428,6 +426,8 @@ const buildMessage = async (msg, cacheKey) => {
         }
       })
     );
+  } else if (msg.content === void 0) {
+    oaiMsg.content = "";
   }
   if (msg.toolCalls && oaiMsg.role === "assistant") {
     oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({

package/dist/llm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this,\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n  label = 'openai.LLMStream';\n\n  constructor(\n    llm: LLM,\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(llm, chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (\n    ((c?: llm.ChatContent | llm.ChatContent[]): c is llm.ChatContent[] => {\n      return (c as llm.ChatContent[]).length !== undefined;\n    })(msg.content)\n  ) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":"AAGA,SAAS,KAAK,WAAW;AACzB,SAAS,kBAAkB;AAC3B,SAAS,aAAa,cAAc;AACpC,OAAO,WAAW;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,OAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,YAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,MAAM,WAAW;AAAA,EACjB,QAAQ;AAAA,EAER,YACEA,MACA,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAMA,MAAK,SAAS,MAAM;AAC1B,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,IAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,IAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,IAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,YACG,CAAC,MAAoE;AACpE,WAAQ,EAAwB,WAAW;AAAA,EAC7C,GAAG,IAAI,OAAO,GACd;AACA,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACC,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,UAAU,MAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["llm","c"]}
+{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this,\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n  label = 'openai.LLMStream';\n\n  constructor(\n    llm: LLM,\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(llm, chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (Array.isArray(msg.content)) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  } else if (msg.content === undefined) {\n    oaiMsg.content = '';\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":"AAGA,SAAS,KAAK,WAAW;AACzB,SAAS,kBAAkB;AAC3B,SAAS,aAAa,cAAc;AACpC,OAAO,WAAW;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,OAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,YAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,MAAM,WAAW;AAAA,EACjB,QAAQ;AAAA,EAER,YACEA,MACA,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAMA,MAAK,SAAS,MAAM;AAC1B,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,IAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,IAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,IAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACC,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF,WAAW,IAAI,YAAY,QAAW;AACpC,WAAO,UAAU;AAAA,EACnB;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,UAAU,MAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["llm","c"]}

package/dist/realtime/realtime_model.cjs

@@ -430,7 +430,8 @@ class RealtimeSession extends import_agents.multimodal.RealtimeSession {
     turnDetection = this.#opts.turnDetection,
     temperature = this.#opts.temperature,
     maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,
-    toolChoice = "auto"
+    toolChoice = "auto",
+    selectedTools = Object.keys(this.#fncCtx || {})
   }) {
     this.#opts = {
       modalities,
@@ -449,7 +450,7 @@ class RealtimeSession extends import_agents.multimodal.RealtimeSession {
       apiVersion: this.#opts.apiVersion,
       entraToken: this.#opts.entraToken
     };
-    const tools = this.#fncCtx ? Object.entries(this.#fncCtx).map(([name, func]) => ({
+    const tools = this.#fncCtx ? Object.entries(this.#fncCtx).filter(([name]) => selectedTools.includes(name)).map(([name, func]) => ({
       type: "function",
       name,
       description: func.description,
@@ -533,7 +534,7 @@ class RealtimeSession extends import_agents.multimodal.RealtimeSession {
       }
       const queryParams = {};
       if (this.#opts.isAzure) {
-        queryParams["api-version"] = "2024-10-01-preview";
+        queryParams["api-version"] = this.#opts.apiVersion ?? "2024-10-01-preview";
         queryParams["deployment"] = this.#opts.model;
       } else {
         queryParams["model"] = this.#opts.model;

package/dist/realtime/realtime_model.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AsyncIterableQueue,\n  Future,\n  Queue,\n  llm,\n  log,\n  mergeFrames,\n  metrics,\n  multimodal,\n} from '@livekit/agents';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { once } from 'node:events';\nimport { WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\ninterface ModelOptions {\n  modalities: ['text', 'audio'] | ['text'];\n  instructions: string;\n  voice: api_proto.Voice;\n  inputAudioFormat: api_proto.AudioFormat;\n  outputAudioFormat: api_proto.AudioFormat;\n  inputAudioTranscription: api_proto.InputAudioTranscription | null;\n  turnDetection: api_proto.TurnDetectionType | null;\n  temperature: number;\n  maxResponseOutputTokens: number;\n  model: api_proto.Model;\n  apiKey?: string;\n  baseURL: string;\n  isAzure: boolean;\n  entraToken?: string;\n  apiVersion?: string;\n}\n\nexport interface RealtimeResponse {\n  id: string;\n  status: api_proto.ResponseStatus;\n  statusDetails: api_proto.ResponseStatusDetails | null;\n  usage: api_proto.ModelUsage | null;\n  output: RealtimeOutput[];\n  doneFut: Future;\n  createdTimestamp: number;\n  firstTokenTimestamp?: number;\n}\n\nexport interface RealtimeOutput {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  role: api_proto.Role;\n  type: 'message' | 'function_call';\n  content: RealtimeContent[];\n  doneFut: Future;\n}\n\nexport interface RealtimeContent {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  contentIndex: number;\n  text: string;\n  audio: AudioFrame[];\n  textStream: AsyncIterableQueue<string>;\n  audioStream: AsyncIterableQueue<AudioFrame>;\n  toolCalls: RealtimeToolCall[];\n  contentType: api_proto.Modality;\n}\n\nexport interface RealtimeToolCall {\n  name: string;\n  arguments: string;\n  toolCallID: string;\n}\n\nexport interface InputSpeechTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n}\n\nexport interface InputSpeechTranscriptionFailed {\n  itemId: string;\n  message: string;\n}\n\nexport interface InputSpeechStarted {\n  itemId: string;\n}\n\nexport interface InputSpeechCommitted {\n  itemId: string;\n}\n\nclass InputAudioBuffer {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  append(frame: AudioFrame) {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.append',\n      audio: Buffer.from(frame.data.buffer).toString('base64'),\n    });\n  }\n\n  clear() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.clear',\n    });\n  }\n\n  commit() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.commit',\n    });\n  }\n}\n\nclass ConversationItem {\n  #session: RealtimeSession;\n  #logger = log();\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  truncate(itemId: string, contentIndex: number, audioEnd: number) {\n    this.#session.queueMsg({\n      type: 'conversation.item.truncate',\n      item_id: itemId,\n      content_index: contentIndex,\n      audio_end_ms: audioEnd,\n    });\n  }\n\n  delete(itemId: string) {\n    this.#session.queueMsg({\n      type: 'conversation.item.delete',\n      item_id: itemId,\n    });\n  }\n\n  create(message: llm.ChatMessage, previousItemId?: string): void {\n    if (!message.content) {\n      return;\n    }\n\n    let event: api_proto.ConversationItemCreateEvent;\n\n    if (message.toolCallId) {\n      if (typeof message.content !== 'string') {\n        throw new TypeError('message.content must be a string');\n      }\n\n      event = {\n        type: 'conversation.item.create',\n        previous_item_id: previousItemId,\n        item: {\n          type: 'function_call_output',\n          call_id: message.toolCallId,\n          output: message.content,\n        },\n      };\n    } else {\n      let content = message.content;\n      if (!Array.isArray(content)) {\n        content = [content];\n      }\n\n      if (message.role === llm.ChatRole.USER) {\n        const contents: (api_proto.InputTextContent | api_proto.InputAudioContent)[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            contents.push({\n              type: 'input_audio',\n              audio: Buffer.from(mergeFrames(c.frame).data.buffer).toString('base64'),\n            });\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'user',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.ASSISTANT) {\n        const contents: api_proto.TextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in assistant message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'assistant',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.SYSTEM) {\n        const contents: api_proto.InputTextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in system message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'system',\n            content: contents,\n          },\n        };\n      } else {\n        this.#logger\n          .child({ message })\n          .warn('chat message is not supported inside the realtime API');\n        return;\n      }\n    }\n\n    this.#session.queueMsg(event);\n  }\n}\n\nclass Conversation {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  get item(): ConversationItem {\n    return new ConversationItem(this.#session);\n  }\n}\n\nclass Response {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  create() {\n    this.#session.queueMsg({\n      type: 'response.create',\n    });\n  }\n\n  cancel() {\n    this.#session.queueMsg({\n      type: 'response.cancel',\n    });\n  }\n}\n\ninterface ContentPtr {\n  response_id: string;\n  output_index: number;\n  content_index: number;\n}\n\nexport class RealtimeModel extends multimodal.RealtimeModel {\n  sampleRate = api_proto.SAMPLE_RATE;\n  numChannels = api_proto.NUM_CHANNELS;\n  inFrameSize = api_proto.IN_FRAME_SIZE;\n  outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n  #defaultOpts: ModelOptions;\n  #sessions: RealtimeSession[] = [];\n\n  static withAzure({\n    baseURL,\n    azureDeployment,\n    apiVersion = '2024-10-01-preview',\n    apiKey = undefined,\n    entraToken = undefined,\n    instructions = '',\n    modalities = ['text', 'audio'],\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n  }: {\n    baseURL: string;\n    azureDeployment: string;\n    apiVersion?: string;\n    apiKey?: string;\n    entraToken?: string;\n    instructions?: string;\n    modalities?: ['text', 'audio'] | ['text'];\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }) {\n    return new RealtimeModel({\n      isAzure: true,\n      baseURL: new URL('openai', baseURL).toString(),\n      model: azureDeployment,\n      apiVersion,\n      apiKey,\n      entraToken,\n      instructions,\n      modalities,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n    });\n  }\n\n  constructor({\n    modalities = ['text', 'audio'],\n    instructions = '',\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n    model = 'gpt-4o-realtime-preview-2024-10-01',\n    apiKey = process.env.OPENAI_API_KEY || '',\n    baseURL = api_proto.BASE_URL,\n    // used for microsoft\n    isAzure = false,\n    apiVersion = undefined,\n    entraToken = undefined,\n  }: {\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    model?: api_proto.Model;\n    apiKey?: string;\n    baseURL?: string;\n    isAzure?: boolean;\n    apiVersion?: string;\n    entraToken?: string;\n  }) {\n    super();\n\n    if (apiKey === '') {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable',\n      );\n    }\n\n    this.#defaultOpts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model,\n      apiKey,\n      baseURL,\n      isAzure,\n      apiVersion,\n      entraToken,\n    };\n  }\n\n  get sessions(): RealtimeSession[] {\n    return this.#sessions;\n  }\n\n  session({\n    fncCtx,\n    chatCtx,\n    modalities = this.#defaultOpts.modalities,\n    instructions = this.#defaultOpts.instructions,\n    voice = this.#defaultOpts.voice,\n    inputAudioFormat = this.#defaultOpts.inputAudioFormat,\n    outputAudioFormat = this.#defaultOpts.outputAudioFormat,\n    inputAudioTranscription = this.#defaultOpts.inputAudioTranscription,\n    turnDetection = this.#defaultOpts.turnDetection,\n    temperature = this.#defaultOpts.temperature,\n    maxResponseOutputTokens = this.#defaultOpts.maxResponseOutputTokens,\n  }: {\n    fncCtx?: llm.FunctionContext;\n    chatCtx?: llm.ChatContext;\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }): RealtimeSession {\n    const opts: ModelOptions = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#defaultOpts.model,\n      apiKey: this.#defaultOpts.apiKey,\n      baseURL: this.#defaultOpts.baseURL,\n      isAzure: this.#defaultOpts.isAzure,\n      apiVersion: this.#defaultOpts.apiVersion,\n      entraToken: this.#defaultOpts.entraToken,\n    };\n\n    const newSession = new RealtimeSession(opts, {\n      chatCtx: chatCtx || new llm.ChatContext(),\n      fncCtx,\n    });\n    this.#sessions.push(newSession);\n    return newSession;\n  }\n\n  async close() {\n    await Promise.allSettled(this.#sessions.map((session) => session.close()));\n  }\n}\n\nexport class RealtimeSession extends multimodal.RealtimeSession {\n  #chatCtx: llm.ChatContext | undefined = undefined;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #opts: ModelOptions;\n  #pendingResponses: { [id: string]: RealtimeResponse } = {};\n  #sessionId = 'not-connected';\n  #ws: WebSocket | null = null;\n  #expiresAt: number | null = null;\n  #logger = log();\n  #task: Promise<void>;\n  #closing = true;\n  #sendQueue = new Queue<api_proto.ClientEvent>();\n\n  constructor(\n    opts: ModelOptions,\n    { fncCtx, chatCtx }: { fncCtx?: llm.FunctionContext; chatCtx?: llm.ChatContext },\n  ) {\n    super();\n\n    this.#opts = opts;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n\n    this.#task = this.#start();\n\n    this.sessionUpdate({\n      modalities: this.#opts.modalities,\n      instructions: this.#opts.instructions,\n      voice: this.#opts.voice,\n      inputAudioFormat: this.#opts.inputAudioFormat,\n      outputAudioFormat: this.#opts.outputAudioFormat,\n      inputAudioTranscription: this.#opts.inputAudioTranscription,\n      turnDetection: this.#opts.turnDetection,\n      temperature: this.#opts.temperature,\n      maxResponseOutputTokens: this.#opts.maxResponseOutputTokens,\n      toolChoice: 'auto',\n    });\n  }\n\n  get chatCtx(): llm.ChatContext | undefined {\n    return this.#chatCtx;\n  }\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n  }\n\n  get conversation(): Conversation {\n    return new Conversation(this);\n  }\n\n  get inputAudioBuffer(): InputAudioBuffer {\n    return new InputAudioBuffer(this);\n  }\n\n  get response(): Response {\n    return new Response(this);\n  }\n\n  get expiration(): number {\n    if (!this.#expiresAt) {\n      throw new Error('session not started');\n    }\n    return this.#expiresAt * 1000;\n  }\n\n  queueMsg(command: api_proto.ClientEvent): void {\n    this.#sendQueue.put(command);\n  }\n\n  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs\n  /// with large amounts of base64 audio data.\n  #loggableEvent(\n    event: api_proto.ClientEvent | api_proto.ServerEvent,\n    maxLength: number = 30,\n  ): Record<string, unknown> {\n    const untypedEvent: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(event)) {\n      if (value !== undefined) {\n        untypedEvent[key] = value;\n      }\n    }\n\n    if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n      const truncatedData =\n        untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? '…' : '');\n      return { ...untypedEvent, audio: truncatedData };\n    }\n    if (\n      untypedEvent.delta &&\n      typeof untypedEvent.delta === 'string' &&\n      event.type === 'response.audio.delta'\n    ) {\n      const truncatedDelta =\n        untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? '…' : '');\n      return { ...untypedEvent, delta: truncatedDelta };\n    }\n    return untypedEvent;\n  }\n\n  sessionUpdate({\n    modalities = this.#opts.modalities,\n    instructions = this.#opts.instructions,\n    voice = this.#opts.voice,\n    inputAudioFormat = this.#opts.inputAudioFormat,\n    outputAudioFormat = this.#opts.outputAudioFormat,\n    inputAudioTranscription = this.#opts.inputAudioTranscription,\n    turnDetection = this.#opts.turnDetection,\n    temperature = this.#opts.temperature,\n    maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,\n    toolChoice = 'auto',\n  }: {\n    modalities: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    toolChoice?: api_proto.ToolChoice;\n  }) {\n    this.#opts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#opts.model,\n      apiKey: this.#opts.apiKey,\n      baseURL: this.#opts.baseURL,\n      isAzure: this.#opts.isAzure,\n      apiVersion: this.#opts.apiVersion,\n      entraToken: this.#opts.entraToken,\n    };\n\n    const tools = this.#fncCtx\n      ? Object.entries(this.#fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          name,\n          description: func.description,\n          parameters:\n            // don't format parameters if they are raw openai params\n            func.parameters.type == ('object' as const)\n              ? func.parameters\n              : llm.oaiParams(func.parameters),\n        }))\n      : [];\n\n    const sessionUpdateEvent: api_proto.SessionUpdateEvent = {\n      type: 'session.update',\n      session: {\n        modalities: this.#opts.modalities,\n        instructions: this.#opts.instructions,\n        voice: this.#opts.voice,\n        input_audio_format: this.#opts.inputAudioFormat,\n        output_audio_format: this.#opts.outputAudioFormat,\n        input_audio_transcription: this.#opts.inputAudioTranscription,\n        turn_detection: this.#opts.turnDetection,\n        temperature: this.#opts.temperature,\n        max_response_output_tokens:\n          this.#opts.maxResponseOutputTokens === Infinity\n            ? 'inf'\n            : this.#opts.maxResponseOutputTokens,\n        tools,\n        tool_choice: toolChoice,\n      },\n    };\n\n    if (this.#opts.isAzure && this.#opts.maxResponseOutputTokens === Infinity) {\n      // microsoft doesn't support inf for max_response_output_tokens, but accepts no args\n      sessionUpdateEvent.session.max_response_output_tokens = undefined;\n    }\n\n    this.queueMsg(sessionUpdateEvent);\n  }\n\n  /** Create an empty audio message with the given duration. */\n  #createEmptyUserAudioMessage(duration: number): llm.ChatMessage {\n    const samples = duration * api_proto.SAMPLE_RATE;\n    return new llm.ChatMessage({\n      role: llm.ChatRole.USER,\n      content: {\n        frame: new AudioFrame(\n          new Int16Array(samples * api_proto.NUM_CHANNELS),\n          api_proto.SAMPLE_RATE,\n          api_proto.NUM_CHANNELS,\n          samples,\n        ),\n      },\n    });\n  }\n\n  /**\n   * Try to recover from a text response to audio mode.\n   *\n   * @remarks\n   * Sometimes the OpenAI Realtime API returns text instead of audio responses.\n   * This method tries to recover from this by requesting a new response after deleting the text\n   * response and creating an empty user audio message.\n   */\n  recoverFromTextResponse(itemId: string) {\n    if (itemId) {\n      this.conversation.item.delete(itemId);\n    }\n    this.conversation.item.create(this.#createEmptyUserAudioMessage(1));\n    this.response.create();\n  }\n\n  #start(): Promise<void> {\n    return new Promise(async (resolve, reject) => {\n      const headers: Record<string, string> = {\n        'User-Agent': 'LiveKit-Agents-JS',\n      };\n      if (this.#opts.isAzure) {\n        // Microsoft API has two ways of authentication\n        // 1. Entra token set as `Bearer` token\n        // 2. API key set as `api_key` header (also accepts query string)\n        if (this.#opts.entraToken) {\n          headers.Authorization = `Bearer ${this.#opts.entraToken}`;\n        } else if (this.#opts.apiKey) {\n          headers['api-key'] = this.#opts.apiKey;\n        } else {\n          reject(new Error('Microsoft API key or entraToken is required'));\n          return;\n        }\n      } else {\n        headers.Authorization = `Bearer ${this.#opts.apiKey}`;\n        headers['OpenAI-Beta'] = 'realtime=v1';\n      }\n      const url = new URL([this.#opts.baseURL, 'realtime'].join('/'));\n      if (url.protocol === 'https:') {\n        url.protocol = 'wss:';\n      }\n\n      // Construct query parameters\n      const queryParams: Record<string, string> = {};\n      if (this.#opts.isAzure) {\n        queryParams['api-version'] = '2024-10-01-preview';\n        queryParams['deployment'] = this.#opts.model;\n      } else {\n        queryParams['model'] = this.#opts.model;\n      }\n\n      for (const [key, value] of Object.entries(queryParams)) {\n        url.searchParams.set(key, value);\n      }\n\n      console.debug('Connecting to OpenAI Realtime API at ', url.toString());\n      this.#ws = new WebSocket(url.toString(), {\n        headers: headers,\n      });\n\n      this.#ws.onerror = (error) => {\n        reject(new Error('OpenAI Realtime WebSocket error: ' + error.message));\n      };\n\n      await once(this.#ws, 'open');\n      this.#closing = false;\n\n      this.#ws.onmessage = (message) => {\n        const event: api_proto.ServerEvent = JSON.parse(message.data as string);\n        this.#logger.debug(`<- ${JSON.stringify(this.#loggableEvent(event))}`);\n        switch (event.type) {\n          case 'error':\n            this.#handleError(event);\n            break;\n          case 'session.created':\n            this.#handleSessionCreated(event);\n            break;\n          case 'session.updated':\n            this.#handleSessionUpdated(event);\n            break;\n          case 'conversation.created':\n            this.#handleConversationCreated(event);\n            break;\n          case 'input_audio_buffer.committed':\n            this.#handleInputAudioBufferCommitted(event);\n            break;\n          case 'input_audio_buffer.cleared':\n            this.#handleInputAudioBufferCleared(event);\n            break;\n          case 'input_audio_buffer.speech_started':\n            this.#handleInputAudioBufferSpeechStarted(event);\n            break;\n          case 'input_audio_buffer.speech_stopped':\n            this.#handleInputAudioBufferSpeechStopped(event);\n            break;\n          case 'conversation.item.created':\n            this.#handleConversationItemCreated(event);\n            break;\n          case 'conversation.item.input_audio_transcription.completed':\n            this.#handleConversationItemInputAudioTranscriptionCompleted(event);\n            break;\n          case 'conversation.item.input_audio_transcription.failed':\n            this.#handleConversationItemInputAudioTranscriptionFailed(event);\n            break;\n          case 'conversation.item.truncated':\n            this.#handleConversationItemTruncated(event);\n            break;\n          case 'conversation.item.deleted':\n            this.#handleConversationItemDeleted(event);\n            break;\n          case 'response.created':\n            this.#handleResponseCreated(event);\n            break;\n          case 'response.done':\n            this.#handleResponseDone(event);\n            break;\n          case 'response.output_item.added':\n            this.#handleResponseOutputItemAdded(event);\n            break;\n          case 'response.output_item.done':\n            this.#handleResponseOutputItemDone(event);\n            break;\n          case 'response.content_part.added':\n            this.#handleResponseContentPartAdded(event);\n            break;\n          case 'response.content_part.done':\n            this.#handleResponseContentPartDone(event);\n            break;\n          case 'response.text.delta':\n            this.#handleResponseTextDelta(event);\n            break;\n          case 'response.text.done':\n            this.#handleResponseTextDone(event);\n            break;\n          case 'response.audio_transcript.delta':\n            this.#handleResponseAudioTranscriptDelta(event);\n            break;\n          case 'response.audio_transcript.done':\n            this.#handleResponseAudioTranscriptDone(event);\n            break;\n          case 'response.audio.delta':\n            this.#handleResponseAudioDelta(event);\n            break;\n          case 'response.audio.done':\n            this.#handleResponseAudioDone(event);\n            break;\n          case 'response.function_call_arguments.delta':\n            this.#handleResponseFunctionCallArgumentsDelta(event);\n            break;\n          case 'response.function_call_arguments.done':\n            this.#handleResponseFunctionCallArgumentsDone(event);\n            break;\n          case 'rate_limits.updated':\n            this.#handleRateLimitsUpdated(event);\n            break;\n        }\n      };\n\n      const sendTask = async () => {\n        while (this.#ws && !this.#closing && this.#ws.readyState === WebSocket.OPEN) {\n          try {\n            const event = await this.#sendQueue.get();\n            if (event.type !== 'input_audio_buffer.append') {\n              this.#logger.debug(`-> ${JSON.stringify(this.#loggableEvent(event))}`);\n            }\n            this.#ws.send(JSON.stringify(event));\n          } catch (error) {\n            this.#logger.error('Error sending event:', error);\n          }\n        }\n      };\n\n      sendTask();\n\n      this.#ws.onclose = () => {\n        if (this.#expiresAt && Date.now() >= this.#expiresAt * 1000) {\n          this.#closing = true;\n        }\n        if (!this.#closing) {\n          reject(new Error('OpenAI Realtime connection closed unexpectedly'));\n        }\n        this.#ws = null;\n        resolve();\n      };\n    });\n  }\n\n  async close() {\n    if (!this.#ws) return;\n    this.#closing = true;\n    this.#ws.close();\n    await this.#task;\n  }\n\n  #getContent(ptr: ContentPtr): RealtimeContent {\n    const response = this.#pendingResponses[ptr.response_id];\n    const output = response!.output[ptr.output_index];\n    const content = output!.content[ptr.content_index]!;\n    return content;\n  }\n\n  #handleError(event: api_proto.ErrorEvent): void {\n    this.#logger.error(`OpenAI Realtime error ${JSON.stringify(event.error)}`);\n  }\n\n  #handleSessionCreated(event: api_proto.SessionCreatedEvent): void {\n    this.#sessionId = event.session.id;\n    this.#expiresAt = event.session.expires_at;\n    this.#logger = this.#logger.child({ sessionId: this.#sessionId });\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleSessionUpdated(event: api_proto.SessionUpdatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationCreated(event: api_proto.ConversationCreatedEvent): void {}\n\n  #handleInputAudioBufferCommitted(event: api_proto.InputAudioBufferCommittedEvent): void {\n    this.emit('input_speech_committed', {\n      itemId: event.item_id,\n    } as InputSpeechCommitted);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleInputAudioBufferCleared(event: api_proto.InputAudioBufferClearedEvent): void {}\n\n  #handleInputAudioBufferSpeechStarted(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStartedEvent,\n  ): void {\n    this.emit('input_speech_started', {\n      itemId: event.item_id,\n    } as InputSpeechStarted);\n  }\n\n  #handleInputAudioBufferSpeechStopped(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStoppedEvent,\n  ): void {\n    this.emit('input_speech_stopped');\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {}\n\n  #handleConversationItemInputAudioTranscriptionCompleted(\n    event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n  ): void {\n    const transcript = event.transcript;\n    this.emit('input_speech_transcription_completed', {\n      itemId: event.item_id,\n      transcript: transcript,\n    } as InputSpeechTranscriptionCompleted);\n  }\n\n  #handleConversationItemInputAudioTranscriptionFailed(\n    event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n  ): void {\n    const error = event.error;\n    this.#logger.error(`OpenAI Realtime failed to transcribe input audio: ${error.message}`);\n    this.emit('input_speech_transcription_failed', {\n      itemId: event.item_id,\n      message: error.message,\n    } as InputSpeechTranscriptionFailed);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemTruncated(event: api_proto.ConversationItemTruncatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {}\n\n  #handleResponseCreated(responseCreated: api_proto.ResponseCreatedEvent): void {\n    const response = responseCreated.response;\n    const doneFut = new Future();\n    const newResponse: RealtimeResponse = {\n      id: response.id,\n      status: response.status,\n      statusDetails: response.status_details,\n      usage: null,\n      output: [],\n      doneFut: doneFut,\n      createdTimestamp: Date.now(),\n    };\n    this.#pendingResponses[newResponse.id] = newResponse;\n    this.emit('response_created', newResponse);\n  }\n\n  #handleResponseDone(event: api_proto.ResponseDoneEvent): void {\n    const responseData = event.response;\n    const responseId = responseData.id;\n    const response = this.#pendingResponses[responseId]!;\n    response.status = responseData.status;\n    response.statusDetails = responseData.status_details;\n    response.usage = responseData.usage ?? null;\n    this.#pendingResponses[responseId] = response;\n    response.doneFut.resolve();\n\n    let metricsError: Error | undefined;\n    let cancelled = false;\n    switch (response.status) {\n      case 'failed': {\n        if (response.statusDetails.type !== 'failed') break;\n        const err = response.statusDetails.error;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          code: err?.code,\n          message: err?.message,\n        });\n        this.#logger\n          .child({ code: err?.code, error: err?.message })\n          .error('response generation failed');\n        break;\n      }\n      case 'incomplete': {\n        if (response.statusDetails.type !== 'incomplete') break;\n        const reason = response.statusDetails.reason;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          reason,\n        });\n        this.#logger.child({ reason }).error('response generation incomplete');\n        break;\n      }\n      case 'cancelled': {\n        cancelled = true;\n        break;\n      }\n    }\n    this.emit('response_done', response);\n\n    let ttft: number | undefined;\n    if (response.firstTokenTimestamp) {\n      ttft = response.firstTokenTimestamp - response.createdTimestamp;\n    }\n    const duration = Date.now() - response.createdTimestamp;\n\n    const usage = response.usage;\n    const metric: metrics.MultimodalLLMMetrics = {\n      timestamp: response.createdTimestamp,\n      requestId: response.id,\n      ttft: ttft!,\n      duration,\n      cancelled,\n      label: this.constructor.name,\n      completionTokens: usage?.output_tokens || 0,\n      promptTokens: usage?.input_tokens || 0,\n      totalTokens: usage?.total_tokens || 0,\n      tokensPerSecond: ((usage?.output_tokens || 0) / duration) * 1000,\n      error: metricsError,\n      inputTokenDetails: {\n        cachedTokens: usage?.input_token_details.cached_tokens || 0,\n        textTokens: usage?.input_token_details.text_tokens || 0,\n        audioTokens: usage?.input_token_details.audio_tokens || 0,\n      },\n      outputTokenDetails: {\n        textTokens: usage?.output_token_details.text_tokens || 0,\n        audioTokens: usage?.output_token_details.audio_tokens || 0,\n      },\n    };\n    this.emit('metrics_collected', metric);\n  }\n\n  #handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const itemData = event.item;\n\n    if (itemData.type !== 'message' && itemData.type !== 'function_call') {\n      throw new Error(`Unexpected item type: ${itemData.type}`);\n    }\n\n    let role: api_proto.Role;\n    if (itemData.type === 'function_call') {\n      role = 'assistant'; // function_call doesn't have a role field, defaulting it to assistant\n    } else {\n      role = itemData.role;\n    }\n\n    const newOutput: RealtimeOutput = {\n      responseId: responseId,\n      itemId: itemData.id,\n      outputIndex: event.output_index,\n      type: itemData.type,\n      role: role,\n      content: [],\n      doneFut: new Future(),\n    };\n    response?.output.push(newOutput);\n    this.emit('response_output_added', newOutput);\n  }\n\n  #handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    if (output?.type === 'function_call') {\n      if (!this.#fncCtx) {\n        this.#logger.error('function call received but no fncCtx is available');\n        return;\n      }\n\n      // parse the arguments and call the function inside the fnc_ctx\n      const item = event.item;\n      if (item.type !== 'function_call') {\n        throw new Error('Expected function_call item');\n      }\n      const func = this.#fncCtx[item.name];\n      if (!func) {\n        this.#logger.error(`no function with name ${item.name} in fncCtx`);\n        return;\n      }\n\n      this.emit('function_call_started', {\n        callId: item.call_id,\n      });\n\n      const parsedArgs = JSON.parse(item.arguments);\n\n      this.#logger.debug(\n        `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`,\n      );\n\n      func.execute(parsedArgs).then(\n        (content) => {\n          this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);\n          this.emit('function_call_completed', {\n            callId: item.call_id,\n          });\n          this.conversation.item.create(\n            llm.ChatMessage.createToolFromFunctionResult({\n              name: item.name,\n              toolCallId: item.call_id,\n              result: content,\n            }),\n            output.itemId,\n          );\n          this.response.create();\n        },\n        (error) => {\n          this.#logger.error(`[Function Call ${item.call_id}] ${item.name} failed with ${error}`);\n          // TODO: send it back up as failed?\n          this.emit('function_call_failed', {\n            callId: item.call_id,\n          });\n        },\n      );\n    }\n\n    output?.doneFut.resolve();\n    this.emit('response_output_done', output);\n  }\n\n  #handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    const textStream = new AsyncIterableQueue<string>();\n    const audioStream = new AsyncIterableQueue<AudioFrame>();\n\n    const newContent: RealtimeContent = {\n      responseId: responseId,\n      itemId: event.item_id,\n      outputIndex: outputIndex,\n      contentIndex: event.content_index,\n      text: '',\n      audio: [],\n      textStream: textStream,\n      audioStream: audioStream,\n      toolCalls: [],\n      contentType: event.part.type,\n    };\n    output?.content.push(newContent);\n    response!.firstTokenTimestamp = Date.now();\n    this.emit('response_content_added', newContent);\n  }\n\n  #handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n    const content = this.#getContent(event);\n    this.emit('response_content_done', content);\n  }\n\n  #handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n    this.emit('response_text_delta', event);\n  }\n\n  #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {\n    const content = this.#getContent(event);\n    content.text = event.text;\n    this.emit('response_text_done', event);\n  }\n\n  #handleResponseAudioTranscriptDelta(event: api_proto.ResponseAudioTranscriptDeltaEvent): void {\n    const content = this.#getContent(event);\n    const transcript = event.delta;\n    content.text += transcript;\n\n    content.textStream.put(transcript);\n  }\n\n  #handleResponseAudioTranscriptDone(event: api_proto.ResponseAudioTranscriptDoneEvent): void {\n    const content = this.#getContent(event);\n    content.textStream.close();\n  }\n\n  #handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n    const content = this.#getContent(event);\n    const data = Buffer.from(event.delta, 'base64');\n    const audio = new AudioFrame(\n      new Int16Array(data.buffer),\n      api_proto.SAMPLE_RATE,\n      api_proto.NUM_CHANNELS,\n      data.length / 2,\n    );\n    content.audio.push(audio);\n\n    content.audioStream.put(audio);\n  }\n\n  #handleResponseAudioDone(event: api_proto.ResponseAudioDoneEvent): void {\n    const content = this.#getContent(event);\n    content.audioStream.close();\n  }\n\n  #handleResponseFunctionCallArgumentsDelta(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDeltaEvent,\n  ): void {}\n\n  #handleResponseFunctionCallArgumentsDone(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDoneEvent,\n  ): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleRateLimitsUpdated(event: api_proto.RateLimitsUpdatedEvent): void {}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBASO;AACP,sBAA2B;AAC3B,yBAAqB;AACrB,gBAA0B;AAC1B,gBAA2B;AA8E3B,MAAM,iBAAiB;AAAA,EACrB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,OAAO,OAAmB;AACxB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,OAAO,OAAO,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,IACzD,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAEA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA,cAAU,mBAAI;AAAA,EAEd,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS,QAAgB,cAAsB,UAAkB;AAC/D,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,MACT,eAAe;AAAA,MACf,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,QAAgB;AACrB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,SAA0B,gBAA+B;AAC9D,QAAI,CAAC,QAAQ,SAAS;AACpB;AAAA,IACF;AAEA,QAAI;AAEJ,QAAI,QAAQ,YAAY;AACtB,UAAI,OAAO,QAAQ,YAAY,UAAU;AACvC,cAAM,IAAI,UAAU,kCAAkC;AAAA,MACxD;AAEA,cAAQ;AAAA,QACN,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,SAAS,QAAQ;AAAA,UACjB,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,IACF,OAAO;AACL,UAAI,UAAU,QAAQ;AACtB,UAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B,kBAAU,CAAC,OAAO;AAAA,MACpB;AAEA,UAAI,QAAQ,SAAS,kBAAI,SAAS,MAAM;AACtC,cAAM,WAAyE,CAAC;AAChF,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,OAAO,OAAO,SAAK,2BAAY,EAAE,KAAK,EAAE,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,kBAAI,SAAS,WAAW;AAClD,cAAM,WAAoC,CAAC;AAC3C,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,qDAAqD;AAAA,UACzE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,kBAAI,SAAS,QAAQ;AAC/C,cAAM,WAAyC,CAAC;AAChD,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,kDAAkD;AAAA,UACtE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,OAAO;AACL,aAAK,QACF,MAAM,EAAE,QAAQ,CAAC,EACjB,KAAK,uDAAuD;AAC/D;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,SAAS,KAAK;AAAA,EAC9B;AACF;AAEA,MAAM,aAAa;AAAA,EACjB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,IAAI,OAAyB;AAC3B,WAAO,IAAI,iBAAiB,KAAK,QAAQ;AAAA,EAC3C;AACF;AAEA,MAAM,SAAS;AAAA,EACb;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAQO,MAAM,sBAAsB,yBAAW,cAAc;AAAA,EAC1D,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA,EAEzB;AAAA,EACA,YAA+B,CAAC;AAAA,EAEhC,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA,aAAa;AAAA,IACb,SAAS;AAAA,IACT,aAAa;AAAA,IACb,eAAe;AAAA,IACf,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,EAC5B,GAeG;AACD,WAAO,IAAI,cAAc;AAAA,MACvB,SAAS;AAAA,MACT,SAAS,IAAI,IAAI,UAAU,OAAO,EAAE,SAAS;AAAA,MAC7C,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,YAAY;AAAA,IACV,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,IAC1B,QAAQ;AAAA,IACR,SAAS,QAAQ,IAAI,kBAAkB;AAAA,IACvC,UAAU,UAAU;AAAA;AAAA,IAEpB,UAAU;AAAA,IACV,aAAa;AAAA,IACb,aAAa;AAAA,EACf,GAgBG;AACD,UAAM;AAEN,QAAI,WAAW,IAAI;AACjB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,SAAK,eAAe;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,WAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AAAA,IACN;AAAA,IACA;AAAA,IACA,aAAa,KAAK,aAAa;AAAA,IAC/B,eAAe,KAAK,aAAa;AAAA,IACjC,QAAQ,KAAK,aAAa;AAAA,IAC1B,mBAAmB,KAAK,aAAa;AAAA,IACrC,oBAAoB,KAAK,aAAa;AAAA,IACtC,0BAA0B,KAAK,aAAa;AAAA,IAC5C,gBAAgB,KAAK,aAAa;AAAA,IAClC,cAAc,KAAK,aAAa;AAAA,IAChC,0BAA0B,KAAK,aAAa;AAAA,EAC9C,GAYoB;AAClB,UAAM,OAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,aAAa;AAAA,MACzB,QAAQ,KAAK,aAAa;AAAA,MAC1B,SAAS,KAAK,aAAa;AAAA,MAC3B,SAAS,KAAK,aAAa;AAAA,MAC3B,YAAY,KAAK,aAAa;AAAA,MAC9B,YAAY,KAAK,aAAa;AAAA,IAChC;AAEA,UAAM,aAAa,IAAI,gBAAgB,MAAM;AAAA,MAC3C,SAAS,WAAW,IAAI,kBAAI,YAAY;AAAA,MACxC;AAAA,IACF,CAAC;AACD,SAAK,UAAU,KAAK,UAAU;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,QAAQ,WAAW,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC3E;AACF;AAEO,MAAM,wBAAwB,yBAAW,gBAAgB;AAAA,EAC9D,WAAwC;AAAA,EACxC,UAA2C;AAAA,EAC3C;AAAA,EACA,oBAAwD,CAAC;AAAA,EACzD,aAAa;AAAA,EACb,MAAwB;AAAA,EACxB,aAA4B;AAAA,EAC5B,cAAU,mBAAI;AAAA,EACd;AAAA,EACA,WAAW;AAAA,EACX,aAAa,IAAI,oBAA6B;AAAA,EAE9C,YACE,MACA,EAAE,QAAQ,QAAQ,GAClB;AACA,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAEf,SAAK,QAAQ,KAAK,OAAO;AAEzB,SAAK,cAAc;AAAA,MACjB,YAAY,KAAK,MAAM;AAAA,MACvB,cAAc,KAAK,MAAM;AAAA,MACzB,OAAO,KAAK,MAAM;AAAA,MAClB,kBAAkB,KAAK,MAAM;AAAA,MAC7B,mBAAmB,KAAK,MAAM;AAAA,MAC9B,yBAAyB,KAAK,MAAM;AAAA,MACpC,eAAe,KAAK,MAAM;AAAA,MAC1B,aAAa,KAAK,MAAM;AAAA,MACxB,yBAAyB,KAAK,MAAM;AAAA,MACpC,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAAA,EAEA,IAAI,UAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,IAAI,eAA6B;AAC/B,WAAO,IAAI,aAAa,IAAI;AAAA,EAC9B;AAAA,EAEA,IAAI,mBAAqC;AACvC,WAAO,IAAI,iBAAiB,IAAI;AAAA,EAClC;AAAA,EAEA,IAAI,WAAqB;AACvB,WAAO,IAAI,SAAS,IAAI;AAAA,EAC1B;AAAA,EAEA,IAAI,aAAqB;AACvB,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,qBAAqB;AAAA,IACvC;AACA,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,SAAS,SAAsC;AAC7C,SAAK,WAAW,IAAI,OAAO;AAAA,EAC7B;AAAA;AAAA;AAAA,EAIA,eACE,OACA,YAAoB,IACK;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,YAAM,gBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,cAAc;AAAA,IACjD;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,YAAM,iBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,eAAe;AAAA,IAClD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc;AAAA,IACZ,aAAa,KAAK,MAAM;AAAA,IACxB,eAAe,KAAK,MAAM;AAAA,IAC1B,QAAQ,KAAK,MAAM;AAAA,IACnB,mBAAmB,KAAK,MAAM;AAAA,IAC9B,oBAAoB,KAAK,MAAM;AAAA,IAC/B,0BAA0B,KAAK,MAAM;AAAA,IACrC,gBAAgB,KAAK,MAAM;AAAA,IAC3B,cAAc,KAAK,MAAM;AAAA,IACzB,0BAA0B,KAAK,MAAM;AAAA,IACrC,aAAa;AAAA,EACf,GAWG;AACD,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,MAAM;AAAA,MAClB,QAAQ,KAAK,MAAM;AAAA,MACnB,SAAS,KAAK,MAAM;AAAA,MACpB,SAAS,KAAK,MAAM;AAAA,MACpB,YAAY,KAAK,MAAM;AAAA,MACvB,YAAY,KAAK,MAAM;AAAA,IACzB;AAEA,UAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MAClD,MAAM;AAAA,MACN;AAAA,MACA,aAAa,KAAK;AAAA,MAClB;AAAA;AAAA,QAEE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,kBAAI,UAAU,KAAK,UAAU;AAAA;AAAA,IACrC,EAAE,IACF,CAAC;AAEL,UAAM,qBAAmD;AAAA,MACvD,MAAM;AAAA,MACN,SAAS;AAAA,QACP,YAAY,KAAK,MAAM;AAAA,QACvB,cAAc,KAAK,MAAM;AAAA,QACzB,OAAO,KAAK,MAAM;AAAA,QAClB,oBAAoB,KAAK,MAAM;AAAA,QAC/B,qBAAqB,KAAK,MAAM;AAAA,QAChC,2BAA2B,KAAK,MAAM;AAAA,QACtC,gBAAgB,KAAK,MAAM;AAAA,QAC3B,aAAa,KAAK,MAAM;AAAA,QACxB,4BACE,KAAK,MAAM,4BAA4B,WACnC,QACA,KAAK,MAAM;AAAA,QACjB;AAAA,QACA,aAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,WAAW,KAAK,MAAM,4BAA4B,UAAU;AAEzE,yBAAmB,QAAQ,6BAA6B;AAAA,IAC1D;AAEA,SAAK,SAAS,kBAAkB;AAAA,EAClC;AAAA;AAAA,EAGA,6BAA6B,UAAmC;AAC9D,UAAM,UAAU,WAAW,UAAU;AACrC,WAAO,IAAI,kBAAI,YAAY;AAAA,MACzB,MAAM,kBAAI,SAAS;AAAA,MACnB,SAAS;AAAA,QACP,OAAO,IAAI;AAAA,UACT,IAAI,WAAW,UAAU,UAAU,YAAY;AAAA,UAC/C,UAAU;AAAA,UACV,UAAU;AAAA,UACV;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,wBAAwB,QAAgB;AACtC,QAAI,QAAQ;AACV,WAAK,aAAa,KAAK,OAAO,MAAM;AAAA,IACtC;AACA,SAAK,aAAa,KAAK,OAAO,KAAK,6BAA6B,CAAC,CAAC;AAClE,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,SAAwB;AACtB,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAC5C,YAAM,UAAkC;AAAA,QACtC,cAAc;AAAA,MAChB;AACA,UAAI,KAAK,MAAM,SAAS;AAItB,YAAI,KAAK,MAAM,YAAY;AACzB,kBAAQ,gBAAgB,UAAU,KAAK,MAAM,UAAU;AAAA,QACzD,WAAW,KAAK,MAAM,QAAQ;AAC5B,kBAAQ,SAAS,IAAI,KAAK,MAAM;AAAA,QAClC,OAAO;AACL,iBAAO,IAAI,MAAM,6CAA6C,CAAC;AAC/D;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,gBAAgB,UAAU,KAAK,MAAM,MAAM;AACnD,gBAAQ,aAAa,IAAI;AAAA,MAC3B;AACA,YAAM,MAAM,IAAI,IAAI,CAAC,KAAK,MAAM,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAC9D,UAAI,IAAI,aAAa,UAAU;AAC7B,YAAI,WAAW;AAAA,MACjB;AAGA,YAAM,cAAsC,CAAC;AAC7C,UAAI,KAAK,MAAM,SAAS;AACtB,oBAAY,aAAa,IAAI;AAC7B,oBAAY,YAAY,IAAI,KAAK,MAAM;AAAA,MACzC,OAAO;AACL,oBAAY,OAAO,IAAI,KAAK,MAAM;AAAA,MACpC;AAEA,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,YAAI,aAAa,IAAI,KAAK,KAAK;AAAA,MACjC;AAEA,cAAQ,MAAM,yCAAyC,IAAI,SAAS,CAAC;AACrE,WAAK,MAAM,IAAI,oBAAU,IAAI,SAAS,GAAG;AAAA,QACvC;AAAA,MACF,CAAC;AAED,WAAK,IAAI,UAAU,CAAC,UAAU;AAC5B,eAAO,IAAI,MAAM,sCAAsC,MAAM,OAAO,CAAC;AAAA,MACvE;AAEA,gBAAM,yBAAK,KAAK,KAAK,MAAM;AAC3B,WAAK,WAAW;AAEhB,WAAK,IAAI,YAAY,CAAC,YAAY;AAChC,cAAM,QAA+B,KAAK,MAAM,QAAQ,IAAc;AACtE,aAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AACrE,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,aAAa,KAAK;AACvB;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,2BAA2B,KAAK;AACrC;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,wDAAwD,KAAK;AAClE;AAAA,UACF,KAAK;AACH,iBAAK,qDAAqD,KAAK;AAC/D;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,uBAAuB,KAAK;AACjC;AAAA,UACF,KAAK;AACH,iBAAK,oBAAoB,KAAK;AAC9B;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,8BAA8B,KAAK;AACxC;AAAA,UACF,KAAK;AACH,iBAAK,gCAAgC,KAAK;AAC1C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,wBAAwB,KAAK;AAClC;AAAA,UACF,KAAK;AACH,iBAAK,oCAAoC,KAAK;AAC9C;AAAA,UACF,KAAK;AACH,iBAAK,mCAAmC,KAAK;AAC7C;AAAA,UACF,KAAK;AACH,iBAAK,0BAA0B,KAAK;AACpC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,0CAA0C,KAAK;AACpD;AAAA,UACF,KAAK;AACH,iBAAK,yCAAyC,KAAK;AACnD;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,QACJ;AAAA,MACF;AAEA,YAAM,WAAW,YAAY;AAC3B,eAAO,KAAK,OAAO,CAAC,KAAK,YAAY,KAAK,IAAI,eAAe,oBAAU,MAAM;AAC3E,cAAI;AACF,kBAAM,QAAQ,MAAM,KAAK,WAAW,IAAI;AACxC,gBAAI,MAAM,SAAS,6BAA6B;AAC9C,mBAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AAAA,YACvE;AACA,iBAAK,IAAI,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,UACrC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,wBAAwB,KAAK;AAAA,UAClD;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAET,WAAK,IAAI,UAAU,MAAM;AACvB,YAAI,KAAK,cAAc,KAAK,IAAI,KAAK,KAAK,aAAa,KAAM;AAC3D,eAAK,WAAW;AAAA,QAClB;AACA,YAAI,CAAC,KAAK,UAAU;AAClB,iBAAO,IAAI,MAAM,gDAAgD,CAAC;AAAA,QACpE;AACA,aAAK,MAAM;AACX,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,CAAC,KAAK,IAAK;AACf,SAAK,WAAW;AAChB,SAAK,IAAI,MAAM;AACf,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,YAAY,KAAkC;AAC5C,UAAM,WAAW,KAAK,kBAAkB,IAAI,WAAW;AACvD,UAAM,SAAS,SAAU,OAAO,IAAI,YAAY;AAChD,UAAM,UAAU,OAAQ,QAAQ,IAAI,aAAa;AACjD,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmC;AAC9C,SAAK,QAAQ,MAAM,yBAAyB,KAAK,UAAU,MAAM,KAAK,CAAC,EAAE;AAAA,EAC3E;AAAA,EAEA,sBAAsB,OAA4C;AAChE,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,UAAU,KAAK,QAAQ,MAAM,EAAE,WAAW,KAAK,WAAW,CAAC;AAAA,EAClE;AAAA;AAAA,EAGA,sBAAsB,OAA4C;AAAA,EAAC;AAAA;AAAA,EAGnE,2BAA2B,OAAiD;AAAA,EAAC;AAAA,EAE7E,iCAAiC,OAAuD;AACtF,SAAK,KAAK,0BAA0B;AAAA,MAClC,QAAQ,MAAM;AAAA,IAChB,CAAyB;AAAA,EAC3B;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,qCAEE,OACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,QAAQ,MAAM;AAAA,IAChB,CAAuB;AAAA,EACzB;AAAA,EAEA,qCAEE,OACM;AACN,SAAK,KAAK,sBAAsB;AAAA,EAClC;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,wDACE,OACM;AACN,UAAM,aAAa,MAAM;AACzB,SAAK,KAAK,wCAAwC;AAAA,MAChD,QAAQ,MAAM;AAAA,MACd;AAAA,IACF,CAAsC;AAAA,EACxC;AAAA,EAEA,qDACE,OACM;AACN,UAAM,QAAQ,MAAM;AACpB,SAAK,QAAQ,MAAM,qDAAqD,MAAM,OAAO,EAAE;AACvF,SAAK,KAAK,qCAAqC;AAAA,MAC7C,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM;AAAA,IACjB,CAAmC;AAAA,EACrC;AAAA;AAAA,EAGA,iCAAiC,OAAuD;AAAA,EAAC;AAAA;AAAA,EAGzF,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,uBAAuB,iBAAuD;AAC5E,UAAM,WAAW,gBAAgB;AACjC,UAAM,UAAU,IAAI,qBAAO;AAC3B,UAAM,cAAgC;AAAA,MACpC,IAAI,SAAS;AAAA,MACb,QAAQ,SAAS;AAAA,MACjB,eAAe,SAAS;AAAA,MACxB,OAAO;AAAA,MACP,QAAQ,CAAC;AAAA,MACT;AAAA,MACA,kBAAkB,KAAK,IAAI;AAAA,IAC7B;AACA,SAAK,kBAAkB,YAAY,EAAE,IAAI;AACzC,SAAK,KAAK,oBAAoB,WAAW;AAAA,EAC3C;AAAA,EAEA,oBAAoB,OAA0C;AAC5D,UAAM,eAAe,MAAM;AAC3B,UAAM,aAAa,aAAa;AAChC,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,aAAS,SAAS,aAAa;AAC/B,aAAS,gBAAgB,aAAa;AACtC,aAAS,QAAQ,aAAa,SAAS;AACvC,SAAK,kBAAkB,UAAU,IAAI;AACrC,aAAS,QAAQ,QAAQ;AAEzB,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,SAAS,QAAQ;AAAA,MACvB,KAAK,UAAU;AACb,YAAI,SAAS,cAAc,SAAS,SAAU;AAC9C,cAAM,MAAM,SAAS,cAAc;AACnC,uBAAe,IAAI,sBAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B,MAAM,2BAAK;AAAA,UACX,SAAS,2BAAK;AAAA,QAChB,CAAC;AACD,aAAK,QACF,MAAM,EAAE,MAAM,2BAAK,MAAM,OAAO,2BAAK,QAAQ,CAAC,EAC9C,MAAM,4BAA4B;AACrC;AAAA,MACF;AAAA,MACA,KAAK,cAAc;AACjB,YAAI,SAAS,cAAc,SAAS,aAAc;AAClD,cAAM,SAAS,SAAS,cAAc;AACtC,uBAAe,IAAI,sBAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B;AAAA,QACF,CAAC;AACD,aAAK,QAAQ,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,gCAAgC;AACrE;AAAA,MACF;AAAA,MACA,KAAK,aAAa;AAChB,oBAAY;AACZ;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAK,iBAAiB,QAAQ;AAEnC,QAAI;AACJ,QAAI,SAAS,qBAAqB;AAChC,aAAO,SAAS,sBAAsB,SAAS;AAAA,IACjD;AACA,UAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AAEvC,UAAM,QAAQ,SAAS;AACvB,UAAM,SAAuC;AAAA,MAC3C,WAAW,SAAS;AAAA,MACpB,WAAW,SAAS;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,YAAY;AAAA,MACxB,mBAAkB,+BAAO,kBAAiB;AAAA,MAC1C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,mBAAmB,+BAAO,kBAAiB,KAAK,WAAY;AAAA,MAC5D,OAAO;AAAA,MACP,mBAAmB;AAAA,QACjB,eAAc,+BAAO,oBAAoB,kBAAiB;AAAA,QAC1D,aAAY,+BAAO,oBAAoB,gBAAe;AAAA,QACtD,cAAa,+BAAO,oBAAoB,iBAAgB;AAAA,MAC1D;AAAA,MACA,oBAAoB;AAAA,QAClB,aAAY,+BAAO,qBAAqB,gBAAe;AAAA,QACvD,cAAa,+BAAO,qBAAqB,iBAAgB;AAAA,MAC3D;AAAA,IACF;AACA,SAAK,KAAK,qBAAqB,MAAM;AAAA,EACvC;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,WAAW,MAAM;AAEvB,QAAI,SAAS,SAAS,aAAa,SAAS,SAAS,iBAAiB;AACpE,YAAM,IAAI,MAAM,yBAAyB,SAAS,IAAI,EAAE;AAAA,IAC1D;AAEA,QAAI;AACJ,QAAI,SAAS,SAAS,iBAAiB;AACrC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,SAAS;AAAA,IAClB;AAEA,UAAM,YAA4B;AAAA,MAChC;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,aAAa,MAAM;AAAA,MACnB,MAAM,SAAS;AAAA,MACf;AAAA,MACA,SAAS,CAAC;AAAA,MACV,SAAS,IAAI,qBAAO;AAAA,IACtB;AACA,yCAAU,OAAO,KAAK;AACtB,SAAK,KAAK,yBAAyB,SAAS;AAAA,EAC9C;AAAA,EAEA,8BAA8B,OAAoD;AAChF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,SAAI,iCAAQ,UAAS,iBAAiB;AACpC,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,QAAQ,MAAM,mDAAmD;AACtE;AAAA,MACF;AAGA,YAAM,OAAO,MAAM;AACnB,UAAI,KAAK,SAAS,iBAAiB;AACjC,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,YAAM,OAAO,KAAK,QAAQ,KAAK,IAAI;AACnC,UAAI,CAAC,MAAM;AACT,aAAK,QAAQ,MAAM,yBAAyB,KAAK,IAAI,YAAY;AACjE;AAAA,MACF;AAEA,WAAK,KAAK,yBAAyB;AAAA,QACjC,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,aAAa,KAAK,MAAM,KAAK,SAAS;AAE5C,WAAK,QAAQ;AAAA,QACX,kBAAkB,KAAK,OAAO,eAAe,KAAK,IAAI,mBAAmB,UAAU;AAAA,MACrF;AAEA,WAAK,QAAQ,UAAU,EAAE;AAAA,QACvB,CAAC,YAAY;AACX,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,aAAa,OAAO,EAAE;AACrF,eAAK,KAAK,2BAA2B;AAAA,YACnC,QAAQ,KAAK;AAAA,UACf,CAAC;AACD,eAAK,aAAa,KAAK;AAAA,YACrB,kBAAI,YAAY,6BAA6B;AAAA,cAC3C,MAAM,KAAK;AAAA,cACX,YAAY,KAAK;AAAA,cACjB,QAAQ;AAAA,YACV,CAAC;AAAA,YACD,OAAO;AAAA,UACT;AACA,eAAK,SAAS,OAAO;AAAA,QACvB;AAAA,QACA,CAAC,UAAU;AACT,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,gBAAgB,KAAK,EAAE;AAEtF,eAAK,KAAK,wBAAwB;AAAA,YAChC,QAAQ,KAAK;AAAA,UACf,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,qCAAQ,QAAQ;AAChB,SAAK,KAAK,wBAAwB,MAAM;AAAA,EAC1C;AAAA,EAEA,gCAAgC,OAAsD;AACpF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,UAAM,aAAa,IAAI,iCAA2B;AAClD,UAAM,cAAc,IAAI,iCAA+B;AAEvD,UAAM,aAA8B;AAAA,MAClC;AAAA,MACA,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,cAAc,MAAM;AAAA,MACpB,MAAM;AAAA,MACN,OAAO,CAAC;AAAA,MACR;AAAA,MACA;AAAA,MACA,WAAW,CAAC;AAAA,MACZ,aAAa,MAAM,KAAK;AAAA,IAC1B;AACA,qCAAQ,QAAQ,KAAK;AACrB,aAAU,sBAAsB,KAAK,IAAI;AACzC,SAAK,KAAK,0BAA0B,UAAU;AAAA,EAChD;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,SAAK,KAAK,yBAAyB,OAAO;AAAA,EAC5C;AAAA,EAEA,yBAAyB,OAA+C;AACtE,SAAK,KAAK,uBAAuB,KAAK;AAAA,EACxC;AAAA,EAEA,wBAAwB,OAA8C;AACpE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,OAAO,MAAM;AACrB,SAAK,KAAK,sBAAsB,KAAK;AAAA,EACvC;AAAA,EAEA,oCAAoC,OAA0D;AAC5F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,aAAa,MAAM;AACzB,YAAQ,QAAQ;AAEhB,YAAQ,WAAW,IAAI,UAAU;AAAA,EACnC;AAAA,EAEA,mCAAmC,OAAyD;AAC1F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,WAAW,MAAM;AAAA,EAC3B;AAAA,EAEA,0BAA0B,OAAgD;AACxE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO,QAAQ;AAC9C,UAAM,QAAQ,IAAI;AAAA,MAChB,IAAI,WAAW,KAAK,MAAM;AAAA,MAC1B,UAAU;AAAA,MACV,UAAU;AAAA,MACV,KAAK,SAAS;AAAA,IAChB;AACA,YAAQ,MAAM,KAAK,KAAK;AAExB,YAAQ,YAAY,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,yBAAyB,OAA+C;AACtE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,YAAY,MAAM;AAAA,EAC5B;AAAA,EAEA,0CAEE,OACM;AAAA,EAAC;AAAA,EAET,yCAEE,OACM;AAAA,EAAC;AAAA;AAAA,EAGT,yBAAyB,OAA+C;AAAA,EAAC;AAC3E;","names":["c"]}
+{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AsyncIterableQueue,\n  Future,\n  Queue,\n  llm,\n  log,\n  mergeFrames,\n  metrics,\n  multimodal,\n} from '@livekit/agents';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { once } from 'node:events';\nimport { WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\ninterface ModelOptions {\n  modalities: ['text', 'audio'] | ['text'];\n  instructions: string;\n  voice: api_proto.Voice;\n  inputAudioFormat: api_proto.AudioFormat;\n  outputAudioFormat: api_proto.AudioFormat;\n  inputAudioTranscription: api_proto.InputAudioTranscription | null;\n  turnDetection: api_proto.TurnDetectionType | null;\n  temperature: number;\n  maxResponseOutputTokens: number;\n  model: api_proto.Model;\n  apiKey?: string;\n  baseURL: string;\n  isAzure: boolean;\n  entraToken?: string;\n  apiVersion?: string;\n}\n\nexport interface RealtimeResponse {\n  id: string;\n  status: api_proto.ResponseStatus;\n  statusDetails: api_proto.ResponseStatusDetails | null;\n  usage: api_proto.ModelUsage | null;\n  output: RealtimeOutput[];\n  doneFut: Future;\n  createdTimestamp: number;\n  firstTokenTimestamp?: number;\n}\n\nexport interface RealtimeOutput {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  role: api_proto.Role;\n  type: 'message' | 'function_call';\n  content: RealtimeContent[];\n  doneFut: Future;\n}\n\nexport interface RealtimeContent {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  contentIndex: number;\n  text: string;\n  audio: AudioFrame[];\n  textStream: AsyncIterableQueue<string>;\n  audioStream: AsyncIterableQueue<AudioFrame>;\n  toolCalls: RealtimeToolCall[];\n  contentType: api_proto.Modality;\n}\n\nexport interface RealtimeToolCall {\n  name: string;\n  arguments: string;\n  toolCallID: string;\n}\n\nexport interface InputSpeechTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n}\n\nexport interface InputSpeechTranscriptionFailed {\n  itemId: string;\n  message: string;\n}\n\nexport interface InputSpeechStarted {\n  itemId: string;\n}\n\nexport interface InputSpeechCommitted {\n  itemId: string;\n}\n\nclass InputAudioBuffer {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  append(frame: AudioFrame) {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.append',\n      audio: Buffer.from(frame.data.buffer).toString('base64'),\n    });\n  }\n\n  clear() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.clear',\n    });\n  }\n\n  commit() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.commit',\n    });\n  }\n}\n\nclass ConversationItem {\n  #session: RealtimeSession;\n  #logger = log();\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  truncate(itemId: string, contentIndex: number, audioEnd: number) {\n    this.#session.queueMsg({\n      type: 'conversation.item.truncate',\n      item_id: itemId,\n      content_index: contentIndex,\n      audio_end_ms: audioEnd,\n    });\n  }\n\n  delete(itemId: string) {\n    this.#session.queueMsg({\n      type: 'conversation.item.delete',\n      item_id: itemId,\n    });\n  }\n\n  create(message: llm.ChatMessage, previousItemId?: string): void {\n    if (!message.content) {\n      return;\n    }\n\n    let event: api_proto.ConversationItemCreateEvent;\n\n    if (message.toolCallId) {\n      if (typeof message.content !== 'string') {\n        throw new TypeError('message.content must be a string');\n      }\n\n      event = {\n        type: 'conversation.item.create',\n        previous_item_id: previousItemId,\n        item: {\n          type: 'function_call_output',\n          call_id: message.toolCallId,\n          output: message.content,\n        },\n      };\n    } else {\n      let content = message.content;\n      if (!Array.isArray(content)) {\n        content = [content];\n      }\n\n      if (message.role === llm.ChatRole.USER) {\n        const contents: (api_proto.InputTextContent | api_proto.InputAudioContent)[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            contents.push({\n              type: 'input_audio',\n              audio: Buffer.from(mergeFrames(c.frame).data.buffer).toString('base64'),\n            });\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'user',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.ASSISTANT) {\n        const contents: api_proto.TextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in assistant message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'assistant',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.SYSTEM) {\n        const contents: api_proto.InputTextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in system message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'system',\n            content: contents,\n          },\n        };\n      } else {\n        this.#logger\n          .child({ message })\n          .warn('chat message is not supported inside the realtime API');\n        return;\n      }\n    }\n\n    this.#session.queueMsg(event);\n  }\n}\n\nclass Conversation {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  get item(): ConversationItem {\n    return new ConversationItem(this.#session);\n  }\n}\n\nclass Response {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  create() {\n    this.#session.queueMsg({\n      type: 'response.create',\n    });\n  }\n\n  cancel() {\n    this.#session.queueMsg({\n      type: 'response.cancel',\n    });\n  }\n}\n\ninterface ContentPtr {\n  response_id: string;\n  output_index: number;\n  content_index: number;\n}\n\nexport class RealtimeModel extends multimodal.RealtimeModel {\n  sampleRate = api_proto.SAMPLE_RATE;\n  numChannels = api_proto.NUM_CHANNELS;\n  inFrameSize = api_proto.IN_FRAME_SIZE;\n  outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n  #defaultOpts: ModelOptions;\n  #sessions: RealtimeSession[] = [];\n\n  static withAzure({\n    baseURL,\n    azureDeployment,\n    apiVersion = '2024-10-01-preview',\n    apiKey = undefined,\n    entraToken = undefined,\n    instructions = '',\n    modalities = ['text', 'audio'],\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n  }: {\n    baseURL: string;\n    azureDeployment: string;\n    apiVersion?: string;\n    apiKey?: string;\n    entraToken?: string;\n    instructions?: string;\n    modalities?: ['text', 'audio'] | ['text'];\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }) {\n    return new RealtimeModel({\n      isAzure: true,\n      baseURL: new URL('openai', baseURL).toString(),\n      model: azureDeployment,\n      apiVersion,\n      apiKey,\n      entraToken,\n      instructions,\n      modalities,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n    });\n  }\n\n  constructor({\n    modalities = ['text', 'audio'],\n    instructions = '',\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n    model = 'gpt-4o-realtime-preview-2024-10-01',\n    apiKey = process.env.OPENAI_API_KEY || '',\n    baseURL = api_proto.BASE_URL,\n    // used for microsoft\n    isAzure = false,\n    apiVersion = undefined,\n    entraToken = undefined,\n  }: {\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    model?: api_proto.Model;\n    apiKey?: string;\n    baseURL?: string;\n    isAzure?: boolean;\n    apiVersion?: string;\n    entraToken?: string;\n  }) {\n    super();\n\n    if (apiKey === '') {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable',\n      );\n    }\n\n    this.#defaultOpts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model,\n      apiKey,\n      baseURL,\n      isAzure,\n      apiVersion,\n      entraToken,\n    };\n  }\n\n  get sessions(): RealtimeSession[] {\n    return this.#sessions;\n  }\n\n  session({\n    fncCtx,\n    chatCtx,\n    modalities = this.#defaultOpts.modalities,\n    instructions = this.#defaultOpts.instructions,\n    voice = this.#defaultOpts.voice,\n    inputAudioFormat = this.#defaultOpts.inputAudioFormat,\n    outputAudioFormat = this.#defaultOpts.outputAudioFormat,\n    inputAudioTranscription = this.#defaultOpts.inputAudioTranscription,\n    turnDetection = this.#defaultOpts.turnDetection,\n    temperature = this.#defaultOpts.temperature,\n    maxResponseOutputTokens = this.#defaultOpts.maxResponseOutputTokens,\n  }: {\n    fncCtx?: llm.FunctionContext;\n    chatCtx?: llm.ChatContext;\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }): RealtimeSession {\n    const opts: ModelOptions = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#defaultOpts.model,\n      apiKey: this.#defaultOpts.apiKey,\n      baseURL: this.#defaultOpts.baseURL,\n      isAzure: this.#defaultOpts.isAzure,\n      apiVersion: this.#defaultOpts.apiVersion,\n      entraToken: this.#defaultOpts.entraToken,\n    };\n\n    const newSession = new RealtimeSession(opts, {\n      chatCtx: chatCtx || new llm.ChatContext(),\n      fncCtx,\n    });\n    this.#sessions.push(newSession);\n    return newSession;\n  }\n\n  async close() {\n    await Promise.allSettled(this.#sessions.map((session) => session.close()));\n  }\n}\n\nexport class RealtimeSession extends multimodal.RealtimeSession {\n  #chatCtx: llm.ChatContext | undefined = undefined;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #opts: ModelOptions;\n  #pendingResponses: { [id: string]: RealtimeResponse } = {};\n  #sessionId = 'not-connected';\n  #ws: WebSocket | null = null;\n  #expiresAt: number | null = null;\n  #logger = log();\n  #task: Promise<void>;\n  #closing = true;\n  #sendQueue = new Queue<api_proto.ClientEvent>();\n\n  constructor(\n    opts: ModelOptions,\n    { fncCtx, chatCtx }: { fncCtx?: llm.FunctionContext; chatCtx?: llm.ChatContext },\n  ) {\n    super();\n\n    this.#opts = opts;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n\n    this.#task = this.#start();\n\n    this.sessionUpdate({\n      modalities: this.#opts.modalities,\n      instructions: this.#opts.instructions,\n      voice: this.#opts.voice,\n      inputAudioFormat: this.#opts.inputAudioFormat,\n      outputAudioFormat: this.#opts.outputAudioFormat,\n      inputAudioTranscription: this.#opts.inputAudioTranscription,\n      turnDetection: this.#opts.turnDetection,\n      temperature: this.#opts.temperature,\n      maxResponseOutputTokens: this.#opts.maxResponseOutputTokens,\n      toolChoice: 'auto',\n    });\n  }\n\n  get chatCtx(): llm.ChatContext | undefined {\n    return this.#chatCtx;\n  }\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n  }\n\n  get conversation(): Conversation {\n    return new Conversation(this);\n  }\n\n  get inputAudioBuffer(): InputAudioBuffer {\n    return new InputAudioBuffer(this);\n  }\n\n  get response(): Response {\n    return new Response(this);\n  }\n\n  get expiration(): number {\n    if (!this.#expiresAt) {\n      throw new Error('session not started');\n    }\n    return this.#expiresAt * 1000;\n  }\n\n  queueMsg(command: api_proto.ClientEvent): void {\n    this.#sendQueue.put(command);\n  }\n\n  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs\n  /// with large amounts of base64 audio data.\n  #loggableEvent(\n    event: api_proto.ClientEvent | api_proto.ServerEvent,\n    maxLength: number = 30,\n  ): Record<string, unknown> {\n    const untypedEvent: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(event)) {\n      if (value !== undefined) {\n        untypedEvent[key] = value;\n      }\n    }\n\n    if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n      const truncatedData =\n        untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? '…' : '');\n      return { ...untypedEvent, audio: truncatedData };\n    }\n    if (\n      untypedEvent.delta &&\n      typeof untypedEvent.delta === 'string' &&\n      event.type === 'response.audio.delta'\n    ) {\n      const truncatedDelta =\n        untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? '…' : '');\n      return { ...untypedEvent, delta: truncatedDelta };\n    }\n    return untypedEvent;\n  }\n\n  sessionUpdate({\n    modalities = this.#opts.modalities,\n    instructions = this.#opts.instructions,\n    voice = this.#opts.voice,\n    inputAudioFormat = this.#opts.inputAudioFormat,\n    outputAudioFormat = this.#opts.outputAudioFormat,\n    inputAudioTranscription = this.#opts.inputAudioTranscription,\n    turnDetection = this.#opts.turnDetection,\n    temperature = this.#opts.temperature,\n    maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,\n    toolChoice = 'auto',\n    selectedTools = Object.keys(this.#fncCtx || {}),\n  }: {\n    modalities: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    toolChoice?: api_proto.ToolChoice;\n    selectedTools?: string[];\n  }) {\n    this.#opts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#opts.model,\n      apiKey: this.#opts.apiKey,\n      baseURL: this.#opts.baseURL,\n      isAzure: this.#opts.isAzure,\n      apiVersion: this.#opts.apiVersion,\n      entraToken: this.#opts.entraToken,\n    };\n\n    const tools = this.#fncCtx\n      ? Object.entries(this.#fncCtx)\n          .filter(([name]) => selectedTools.includes(name))\n          .map(([name, func]) => ({\n            type: 'function' as const,\n            name,\n            description: func.description,\n            parameters:\n              // don't format parameters if they are raw openai params\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          }))\n      : [];\n\n    const sessionUpdateEvent: api_proto.SessionUpdateEvent = {\n      type: 'session.update',\n      session: {\n        modalities: this.#opts.modalities,\n        instructions: this.#opts.instructions,\n        voice: this.#opts.voice,\n        input_audio_format: this.#opts.inputAudioFormat,\n        output_audio_format: this.#opts.outputAudioFormat,\n        input_audio_transcription: this.#opts.inputAudioTranscription,\n        turn_detection: this.#opts.turnDetection,\n        temperature: this.#opts.temperature,\n        max_response_output_tokens:\n          this.#opts.maxResponseOutputTokens === Infinity\n            ? 'inf'\n            : this.#opts.maxResponseOutputTokens,\n        tools,\n        tool_choice: toolChoice,\n      },\n    };\n\n    if (this.#opts.isAzure && this.#opts.maxResponseOutputTokens === Infinity) {\n      // microsoft doesn't support inf for max_response_output_tokens, but accepts no args\n      sessionUpdateEvent.session.max_response_output_tokens = undefined;\n    }\n\n    this.queueMsg(sessionUpdateEvent);\n  }\n\n  /** Create an empty audio message with the given duration. */\n  #createEmptyUserAudioMessage(duration: number): llm.ChatMessage {\n    const samples = duration * api_proto.SAMPLE_RATE;\n    return new llm.ChatMessage({\n      role: llm.ChatRole.USER,\n      content: {\n        frame: new AudioFrame(\n          new Int16Array(samples * api_proto.NUM_CHANNELS),\n          api_proto.SAMPLE_RATE,\n          api_proto.NUM_CHANNELS,\n          samples,\n        ),\n      },\n    });\n  }\n\n  /**\n   * Try to recover from a text response to audio mode.\n   *\n   * @remarks\n   * Sometimes the OpenAI Realtime API returns text instead of audio responses.\n   * This method tries to recover from this by requesting a new response after deleting the text\n   * response and creating an empty user audio message.\n   */\n  recoverFromTextResponse(itemId: string) {\n    if (itemId) {\n      this.conversation.item.delete(itemId);\n    }\n    this.conversation.item.create(this.#createEmptyUserAudioMessage(1));\n    this.response.create();\n  }\n\n  #start(): Promise<void> {\n    return new Promise(async (resolve, reject) => {\n      const headers: Record<string, string> = {\n        'User-Agent': 'LiveKit-Agents-JS',\n      };\n      if (this.#opts.isAzure) {\n        // Microsoft API has two ways of authentication\n        // 1. Entra token set as `Bearer` token\n        // 2. API key set as `api_key` header (also accepts query string)\n        if (this.#opts.entraToken) {\n          headers.Authorization = `Bearer ${this.#opts.entraToken}`;\n        } else if (this.#opts.apiKey) {\n          headers['api-key'] = this.#opts.apiKey;\n        } else {\n          reject(new Error('Microsoft API key or entraToken is required'));\n          return;\n        }\n      } else {\n        headers.Authorization = `Bearer ${this.#opts.apiKey}`;\n        headers['OpenAI-Beta'] = 'realtime=v1';\n      }\n      const url = new URL([this.#opts.baseURL, 'realtime'].join('/'));\n      if (url.protocol === 'https:') {\n        url.protocol = 'wss:';\n      }\n\n      // Construct query parameters\n      const queryParams: Record<string, string> = {};\n      if (this.#opts.isAzure) {\n        queryParams['api-version'] = this.#opts.apiVersion ?? '2024-10-01-preview';\n        queryParams['deployment'] = this.#opts.model;\n      } else {\n        queryParams['model'] = this.#opts.model;\n      }\n\n      for (const [key, value] of Object.entries(queryParams)) {\n        url.searchParams.set(key, value);\n      }\n\n      console.debug('Connecting to OpenAI Realtime API at ', url.toString());\n      this.#ws = new WebSocket(url.toString(), {\n        headers: headers,\n      });\n\n      this.#ws.onerror = (error) => {\n        reject(new Error('OpenAI Realtime WebSocket error: ' + error.message));\n      };\n\n      await once(this.#ws, 'open');\n      this.#closing = false;\n\n      this.#ws.onmessage = (message) => {\n        const event: api_proto.ServerEvent = JSON.parse(message.data as string);\n        this.#logger.debug(`<- ${JSON.stringify(this.#loggableEvent(event))}`);\n        switch (event.type) {\n          case 'error':\n            this.#handleError(event);\n            break;\n          case 'session.created':\n            this.#handleSessionCreated(event);\n            break;\n          case 'session.updated':\n            this.#handleSessionUpdated(event);\n            break;\n          case 'conversation.created':\n            this.#handleConversationCreated(event);\n            break;\n          case 'input_audio_buffer.committed':\n            this.#handleInputAudioBufferCommitted(event);\n            break;\n          case 'input_audio_buffer.cleared':\n            this.#handleInputAudioBufferCleared(event);\n            break;\n          case 'input_audio_buffer.speech_started':\n            this.#handleInputAudioBufferSpeechStarted(event);\n            break;\n          case 'input_audio_buffer.speech_stopped':\n            this.#handleInputAudioBufferSpeechStopped(event);\n            break;\n          case 'conversation.item.created':\n            this.#handleConversationItemCreated(event);\n            break;\n          case 'conversation.item.input_audio_transcription.completed':\n            this.#handleConversationItemInputAudioTranscriptionCompleted(event);\n            break;\n          case 'conversation.item.input_audio_transcription.failed':\n            this.#handleConversationItemInputAudioTranscriptionFailed(event);\n            break;\n          case 'conversation.item.truncated':\n            this.#handleConversationItemTruncated(event);\n            break;\n          case 'conversation.item.deleted':\n            this.#handleConversationItemDeleted(event);\n            break;\n          case 'response.created':\n            this.#handleResponseCreated(event);\n            break;\n          case 'response.done':\n            this.#handleResponseDone(event);\n            break;\n          case 'response.output_item.added':\n            this.#handleResponseOutputItemAdded(event);\n            break;\n          case 'response.output_item.done':\n            this.#handleResponseOutputItemDone(event);\n            break;\n          case 'response.content_part.added':\n            this.#handleResponseContentPartAdded(event);\n            break;\n          case 'response.content_part.done':\n            this.#handleResponseContentPartDone(event);\n            break;\n          case 'response.text.delta':\n            this.#handleResponseTextDelta(event);\n            break;\n          case 'response.text.done':\n            this.#handleResponseTextDone(event);\n            break;\n          case 'response.audio_transcript.delta':\n            this.#handleResponseAudioTranscriptDelta(event);\n            break;\n          case 'response.audio_transcript.done':\n            this.#handleResponseAudioTranscriptDone(event);\n            break;\n          case 'response.audio.delta':\n            this.#handleResponseAudioDelta(event);\n            break;\n          case 'response.audio.done':\n            this.#handleResponseAudioDone(event);\n            break;\n          case 'response.function_call_arguments.delta':\n            this.#handleResponseFunctionCallArgumentsDelta(event);\n            break;\n          case 'response.function_call_arguments.done':\n            this.#handleResponseFunctionCallArgumentsDone(event);\n            break;\n          case 'rate_limits.updated':\n            this.#handleRateLimitsUpdated(event);\n            break;\n        }\n      };\n\n      const sendTask = async () => {\n        while (this.#ws && !this.#closing && this.#ws.readyState === WebSocket.OPEN) {\n          try {\n            const event = await this.#sendQueue.get();\n            if (event.type !== 'input_audio_buffer.append') {\n              this.#logger.debug(`-> ${JSON.stringify(this.#loggableEvent(event))}`);\n            }\n            this.#ws.send(JSON.stringify(event));\n          } catch (error) {\n            this.#logger.error('Error sending event:', error);\n          }\n        }\n      };\n\n      sendTask();\n\n      this.#ws.onclose = () => {\n        if (this.#expiresAt && Date.now() >= this.#expiresAt * 1000) {\n          this.#closing = true;\n        }\n        if (!this.#closing) {\n          reject(new Error('OpenAI Realtime connection closed unexpectedly'));\n        }\n        this.#ws = null;\n        resolve();\n      };\n    });\n  }\n\n  async close() {\n    if (!this.#ws) return;\n    this.#closing = true;\n    this.#ws.close();\n    await this.#task;\n  }\n\n  #getContent(ptr: ContentPtr): RealtimeContent {\n    const response = this.#pendingResponses[ptr.response_id];\n    const output = response!.output[ptr.output_index];\n    const content = output!.content[ptr.content_index]!;\n    return content;\n  }\n\n  #handleError(event: api_proto.ErrorEvent): void {\n    this.#logger.error(`OpenAI Realtime error ${JSON.stringify(event.error)}`);\n  }\n\n  #handleSessionCreated(event: api_proto.SessionCreatedEvent): void {\n    this.#sessionId = event.session.id;\n    this.#expiresAt = event.session.expires_at;\n    this.#logger = this.#logger.child({ sessionId: this.#sessionId });\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleSessionUpdated(event: api_proto.SessionUpdatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationCreated(event: api_proto.ConversationCreatedEvent): void {}\n\n  #handleInputAudioBufferCommitted(event: api_proto.InputAudioBufferCommittedEvent): void {\n    this.emit('input_speech_committed', {\n      itemId: event.item_id,\n    } as InputSpeechCommitted);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleInputAudioBufferCleared(event: api_proto.InputAudioBufferClearedEvent): void {}\n\n  #handleInputAudioBufferSpeechStarted(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStartedEvent,\n  ): void {\n    this.emit('input_speech_started', {\n      itemId: event.item_id,\n    } as InputSpeechStarted);\n  }\n\n  #handleInputAudioBufferSpeechStopped(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStoppedEvent,\n  ): void {\n    this.emit('input_speech_stopped');\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {}\n\n  #handleConversationItemInputAudioTranscriptionCompleted(\n    event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n  ): void {\n    const transcript = event.transcript;\n    this.emit('input_speech_transcription_completed', {\n      itemId: event.item_id,\n      transcript: transcript,\n    } as InputSpeechTranscriptionCompleted);\n  }\n\n  #handleConversationItemInputAudioTranscriptionFailed(\n    event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n  ): void {\n    const error = event.error;\n    this.#logger.error(`OpenAI Realtime failed to transcribe input audio: ${error.message}`);\n    this.emit('input_speech_transcription_failed', {\n      itemId: event.item_id,\n      message: error.message,\n    } as InputSpeechTranscriptionFailed);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemTruncated(event: api_proto.ConversationItemTruncatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {}\n\n  #handleResponseCreated(responseCreated: api_proto.ResponseCreatedEvent): void {\n    const response = responseCreated.response;\n    const doneFut = new Future();\n    const newResponse: RealtimeResponse = {\n      id: response.id,\n      status: response.status,\n      statusDetails: response.status_details,\n      usage: null,\n      output: [],\n      doneFut: doneFut,\n      createdTimestamp: Date.now(),\n    };\n    this.#pendingResponses[newResponse.id] = newResponse;\n    this.emit('response_created', newResponse);\n  }\n\n  #handleResponseDone(event: api_proto.ResponseDoneEvent): void {\n    const responseData = event.response;\n    const responseId = responseData.id;\n    const response = this.#pendingResponses[responseId]!;\n    response.status = responseData.status;\n    response.statusDetails = responseData.status_details;\n    response.usage = responseData.usage ?? null;\n    this.#pendingResponses[responseId] = response;\n    response.doneFut.resolve();\n\n    let metricsError: Error | undefined;\n    let cancelled = false;\n    switch (response.status) {\n      case 'failed': {\n        if (response.statusDetails.type !== 'failed') break;\n        const err = response.statusDetails.error;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          code: err?.code,\n          message: err?.message,\n        });\n        this.#logger\n          .child({ code: err?.code, error: err?.message })\n          .error('response generation failed');\n        break;\n      }\n      case 'incomplete': {\n        if (response.statusDetails.type !== 'incomplete') break;\n        const reason = response.statusDetails.reason;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          reason,\n        });\n        this.#logger.child({ reason }).error('response generation incomplete');\n        break;\n      }\n      case 'cancelled': {\n        cancelled = true;\n        break;\n      }\n    }\n    this.emit('response_done', response);\n\n    let ttft: number | undefined;\n    if (response.firstTokenTimestamp) {\n      ttft = response.firstTokenTimestamp - response.createdTimestamp;\n    }\n    const duration = Date.now() - response.createdTimestamp;\n\n    const usage = response.usage;\n    const metric: metrics.MultimodalLLMMetrics = {\n      timestamp: response.createdTimestamp,\n      requestId: response.id,\n      ttft: ttft!,\n      duration,\n      cancelled,\n      label: this.constructor.name,\n      completionTokens: usage?.output_tokens || 0,\n      promptTokens: usage?.input_tokens || 0,\n      totalTokens: usage?.total_tokens || 0,\n      tokensPerSecond: ((usage?.output_tokens || 0) / duration) * 1000,\n      error: metricsError,\n      inputTokenDetails: {\n        cachedTokens: usage?.input_token_details.cached_tokens || 0,\n        textTokens: usage?.input_token_details.text_tokens || 0,\n        audioTokens: usage?.input_token_details.audio_tokens || 0,\n      },\n      outputTokenDetails: {\n        textTokens: usage?.output_token_details.text_tokens || 0,\n        audioTokens: usage?.output_token_details.audio_tokens || 0,\n      },\n    };\n    this.emit('metrics_collected', metric);\n  }\n\n  #handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const itemData = event.item;\n\n    if (itemData.type !== 'message' && itemData.type !== 'function_call') {\n      throw new Error(`Unexpected item type: ${itemData.type}`);\n    }\n\n    let role: api_proto.Role;\n    if (itemData.type === 'function_call') {\n      role = 'assistant'; // function_call doesn't have a role field, defaulting it to assistant\n    } else {\n      role = itemData.role;\n    }\n\n    const newOutput: RealtimeOutput = {\n      responseId: responseId,\n      itemId: itemData.id,\n      outputIndex: event.output_index,\n      type: itemData.type,\n      role: role,\n      content: [],\n      doneFut: new Future(),\n    };\n    response?.output.push(newOutput);\n    this.emit('response_output_added', newOutput);\n  }\n\n  #handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    if (output?.type === 'function_call') {\n      if (!this.#fncCtx) {\n        this.#logger.error('function call received but no fncCtx is available');\n        return;\n      }\n\n      // parse the arguments and call the function inside the fnc_ctx\n      const item = event.item;\n      if (item.type !== 'function_call') {\n        throw new Error('Expected function_call item');\n      }\n      const func = this.#fncCtx[item.name];\n      if (!func) {\n        this.#logger.error(`no function with name ${item.name} in fncCtx`);\n        return;\n      }\n\n      this.emit('function_call_started', {\n        callId: item.call_id,\n      });\n\n      const parsedArgs = JSON.parse(item.arguments);\n\n      this.#logger.debug(\n        `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`,\n      );\n\n      func.execute(parsedArgs).then(\n        (content) => {\n          this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);\n          this.emit('function_call_completed', {\n            callId: item.call_id,\n          });\n          this.conversation.item.create(\n            llm.ChatMessage.createToolFromFunctionResult({\n              name: item.name,\n              toolCallId: item.call_id,\n              result: content,\n            }),\n            output.itemId,\n          );\n          this.response.create();\n        },\n        (error) => {\n          this.#logger.error(`[Function Call ${item.call_id}] ${item.name} failed with ${error}`);\n          // TODO: send it back up as failed?\n          this.emit('function_call_failed', {\n            callId: item.call_id,\n          });\n        },\n      );\n    }\n\n    output?.doneFut.resolve();\n    this.emit('response_output_done', output);\n  }\n\n  #handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    const textStream = new AsyncIterableQueue<string>();\n    const audioStream = new AsyncIterableQueue<AudioFrame>();\n\n    const newContent: RealtimeContent = {\n      responseId: responseId,\n      itemId: event.item_id,\n      outputIndex: outputIndex,\n      contentIndex: event.content_index,\n      text: '',\n      audio: [],\n      textStream: textStream,\n      audioStream: audioStream,\n      toolCalls: [],\n      contentType: event.part.type,\n    };\n    output?.content.push(newContent);\n    response!.firstTokenTimestamp = Date.now();\n    this.emit('response_content_added', newContent);\n  }\n\n  #handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n    const content = this.#getContent(event);\n    this.emit('response_content_done', content);\n  }\n\n  #handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n    this.emit('response_text_delta', event);\n  }\n\n  #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {\n    const content = this.#getContent(event);\n    content.text = event.text;\n    this.emit('response_text_done', event);\n  }\n\n  #handleResponseAudioTranscriptDelta(event: api_proto.ResponseAudioTranscriptDeltaEvent): void {\n    const content = this.#getContent(event);\n    const transcript = event.delta;\n    content.text += transcript;\n\n    content.textStream.put(transcript);\n  }\n\n  #handleResponseAudioTranscriptDone(event: api_proto.ResponseAudioTranscriptDoneEvent): void {\n    const content = this.#getContent(event);\n    content.textStream.close();\n  }\n\n  #handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n    const content = this.#getContent(event);\n    const data = Buffer.from(event.delta, 'base64');\n    const audio = new AudioFrame(\n      new Int16Array(data.buffer),\n      api_proto.SAMPLE_RATE,\n      api_proto.NUM_CHANNELS,\n      data.length / 2,\n    );\n    content.audio.push(audio);\n\n    content.audioStream.put(audio);\n  }\n\n  #handleResponseAudioDone(event: api_proto.ResponseAudioDoneEvent): void {\n    const content = this.#getContent(event);\n    content.audioStream.close();\n  }\n\n  #handleResponseFunctionCallArgumentsDelta(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDeltaEvent,\n  ): void {}\n\n  #handleResponseFunctionCallArgumentsDone(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDoneEvent,\n  ): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleRateLimitsUpdated(event: api_proto.RateLimitsUpdatedEvent): void {}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBASO;AACP,sBAA2B;AAC3B,yBAAqB;AACrB,gBAA0B;AAC1B,gBAA2B;AA8E3B,MAAM,iBAAiB;AAAA,EACrB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,OAAO,OAAmB;AACxB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,OAAO,OAAO,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,IACzD,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAEA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA,cAAU,mBAAI;AAAA,EAEd,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS,QAAgB,cAAsB,UAAkB;AAC/D,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,MACT,eAAe;AAAA,MACf,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,QAAgB;AACrB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,SAA0B,gBAA+B;AAC9D,QAAI,CAAC,QAAQ,SAAS;AACpB;AAAA,IACF;AAEA,QAAI;AAEJ,QAAI,QAAQ,YAAY;AACtB,UAAI,OAAO,QAAQ,YAAY,UAAU;AACvC,cAAM,IAAI,UAAU,kCAAkC;AAAA,MACxD;AAEA,cAAQ;AAAA,QACN,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,SAAS,QAAQ;AAAA,UACjB,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,IACF,OAAO;AACL,UAAI,UAAU,QAAQ;AACtB,UAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B,kBAAU,CAAC,OAAO;AAAA,MACpB;AAEA,UAAI,QAAQ,SAAS,kBAAI,SAAS,MAAM;AACtC,cAAM,WAAyE,CAAC;AAChF,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,OAAO,OAAO,SAAK,2BAAY,EAAE,KAAK,EAAE,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,kBAAI,SAAS,WAAW;AAClD,cAAM,WAAoC,CAAC;AAC3C,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,qDAAqD;AAAA,UACzE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,kBAAI,SAAS,QAAQ;AAC/C,cAAM,WAAyC,CAAC;AAChD,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,kDAAkD;AAAA,UACtE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,OAAO;AACL,aAAK,QACF,MAAM,EAAE,QAAQ,CAAC,EACjB,KAAK,uDAAuD;AAC/D;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,SAAS,KAAK;AAAA,EAC9B;AACF;AAEA,MAAM,aAAa;AAAA,EACjB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,IAAI,OAAyB;AAC3B,WAAO,IAAI,iBAAiB,KAAK,QAAQ;AAAA,EAC3C;AACF;AAEA,MAAM,SAAS;AAAA,EACb;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAQO,MAAM,sBAAsB,yBAAW,cAAc;AAAA,EAC1D,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA,EAEzB;AAAA,EACA,YAA+B,CAAC;AAAA,EAEhC,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA,aAAa;AAAA,IACb,SAAS;AAAA,IACT,aAAa;AAAA,IACb,eAAe;AAAA,IACf,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,EAC5B,GAeG;AACD,WAAO,IAAI,cAAc;AAAA,MACvB,SAAS;AAAA,MACT,SAAS,IAAI,IAAI,UAAU,OAAO,EAAE,SAAS;AAAA,MAC7C,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,YAAY;AAAA,IACV,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,IAC1B,QAAQ;AAAA,IACR,SAAS,QAAQ,IAAI,kBAAkB;AAAA,IACvC,UAAU,UAAU;AAAA;AAAA,IAEpB,UAAU;AAAA,IACV,aAAa;AAAA,IACb,aAAa;AAAA,EACf,GAgBG;AACD,UAAM;AAEN,QAAI,WAAW,IAAI;AACjB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,SAAK,eAAe;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,WAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AAAA,IACN;AAAA,IACA;AAAA,IACA,aAAa,KAAK,aAAa;AAAA,IAC/B,eAAe,KAAK,aAAa;AAAA,IACjC,QAAQ,KAAK,aAAa;AAAA,IAC1B,mBAAmB,KAAK,aAAa;AAAA,IACrC,oBAAoB,KAAK,aAAa;AAAA,IACtC,0BAA0B,KAAK,aAAa;AAAA,IAC5C,gBAAgB,KAAK,aAAa;AAAA,IAClC,cAAc,KAAK,aAAa;AAAA,IAChC,0BAA0B,KAAK,aAAa;AAAA,EAC9C,GAYoB;AAClB,UAAM,OAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,aAAa;AAAA,MACzB,QAAQ,KAAK,aAAa;AAAA,MAC1B,SAAS,KAAK,aAAa;AAAA,MAC3B,SAAS,KAAK,aAAa;AAAA,MAC3B,YAAY,KAAK,aAAa;AAAA,MAC9B,YAAY,KAAK,aAAa;AAAA,IAChC;AAEA,UAAM,aAAa,IAAI,gBAAgB,MAAM;AAAA,MAC3C,SAAS,WAAW,IAAI,kBAAI,YAAY;AAAA,MACxC;AAAA,IACF,CAAC;AACD,SAAK,UAAU,KAAK,UAAU;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,QAAQ,WAAW,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC3E;AACF;AAEO,MAAM,wBAAwB,yBAAW,gBAAgB;AAAA,EAC9D,WAAwC;AAAA,EACxC,UAA2C;AAAA,EAC3C;AAAA,EACA,oBAAwD,CAAC;AAAA,EACzD,aAAa;AAAA,EACb,MAAwB;AAAA,EACxB,aAA4B;AAAA,EAC5B,cAAU,mBAAI;AAAA,EACd;AAAA,EACA,WAAW;AAAA,EACX,aAAa,IAAI,oBAA6B;AAAA,EAE9C,YACE,MACA,EAAE,QAAQ,QAAQ,GAClB;AACA,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAEf,SAAK,QAAQ,KAAK,OAAO;AAEzB,SAAK,cAAc;AAAA,MACjB,YAAY,KAAK,MAAM;AAAA,MACvB,cAAc,KAAK,MAAM;AAAA,MACzB,OAAO,KAAK,MAAM;AAAA,MAClB,kBAAkB,KAAK,MAAM;AAAA,MAC7B,mBAAmB,KAAK,MAAM;AAAA,MAC9B,yBAAyB,KAAK,MAAM;AAAA,MACpC,eAAe,KAAK,MAAM;AAAA,MAC1B,aAAa,KAAK,MAAM;AAAA,MACxB,yBAAyB,KAAK,MAAM;AAAA,MACpC,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAAA,EAEA,IAAI,UAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,IAAI,eAA6B;AAC/B,WAAO,IAAI,aAAa,IAAI;AAAA,EAC9B;AAAA,EAEA,IAAI,mBAAqC;AACvC,WAAO,IAAI,iBAAiB,IAAI;AAAA,EAClC;AAAA,EAEA,IAAI,WAAqB;AACvB,WAAO,IAAI,SAAS,IAAI;AAAA,EAC1B;AAAA,EAEA,IAAI,aAAqB;AACvB,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,qBAAqB;AAAA,IACvC;AACA,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,SAAS,SAAsC;AAC7C,SAAK,WAAW,IAAI,OAAO;AAAA,EAC7B;AAAA;AAAA;AAAA,EAIA,eACE,OACA,YAAoB,IACK;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,YAAM,gBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,cAAc;AAAA,IACjD;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,YAAM,iBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,eAAe;AAAA,IAClD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc;AAAA,IACZ,aAAa,KAAK,MAAM;AAAA,IACxB,eAAe,KAAK,MAAM;AAAA,IAC1B,QAAQ,KAAK,MAAM;AAAA,IACnB,mBAAmB,KAAK,MAAM;AAAA,IAC9B,oBAAoB,KAAK,MAAM;AAAA,IAC/B,0BAA0B,KAAK,MAAM;AAAA,IACrC,gBAAgB,KAAK,MAAM;AAAA,IAC3B,cAAc,KAAK,MAAM;AAAA,IACzB,0BAA0B,KAAK,MAAM;AAAA,IACrC,aAAa;AAAA,IACb,gBAAgB,OAAO,KAAK,KAAK,WAAW,CAAC,CAAC;AAAA,EAChD,GAYG;AACD,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,MAAM;AAAA,MAClB,QAAQ,KAAK,MAAM;AAAA,MACnB,SAAS,KAAK,MAAM;AAAA,MACpB,SAAS,KAAK,MAAM;AAAA,MACpB,YAAY,KAAK,MAAM;AAAA,MACvB,YAAY,KAAK,MAAM;AAAA,IACzB;AAEA,UAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EACxB,OAAO,CAAC,CAAC,IAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACtB,MAAM;AAAA,MACN;AAAA,MACA,aAAa,KAAK;AAAA,MAClB;AAAA;AAAA,QAEE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,kBAAI,UAAU,KAAK,UAAU;AAAA;AAAA,IACrC,EAAE,IACJ,CAAC;AAEL,UAAM,qBAAmD;AAAA,MACvD,MAAM;AAAA,MACN,SAAS;AAAA,QACP,YAAY,KAAK,MAAM;AAAA,QACvB,cAAc,KAAK,MAAM;AAAA,QACzB,OAAO,KAAK,MAAM;AAAA,QAClB,oBAAoB,KAAK,MAAM;AAAA,QAC/B,qBAAqB,KAAK,MAAM;AAAA,QAChC,2BAA2B,KAAK,MAAM;AAAA,QACtC,gBAAgB,KAAK,MAAM;AAAA,QAC3B,aAAa,KAAK,MAAM;AAAA,QACxB,4BACE,KAAK,MAAM,4BAA4B,WACnC,QACA,KAAK,MAAM;AAAA,QACjB;AAAA,QACA,aAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,WAAW,KAAK,MAAM,4BAA4B,UAAU;AAEzE,yBAAmB,QAAQ,6BAA6B;AAAA,IAC1D;AAEA,SAAK,SAAS,kBAAkB;AAAA,EAClC;AAAA;AAAA,EAGA,6BAA6B,UAAmC;AAC9D,UAAM,UAAU,WAAW,UAAU;AACrC,WAAO,IAAI,kBAAI,YAAY;AAAA,MACzB,MAAM,kBAAI,SAAS;AAAA,MACnB,SAAS;AAAA,QACP,OAAO,IAAI;AAAA,UACT,IAAI,WAAW,UAAU,UAAU,YAAY;AAAA,UAC/C,UAAU;AAAA,UACV,UAAU;AAAA,UACV;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,wBAAwB,QAAgB;AACtC,QAAI,QAAQ;AACV,WAAK,aAAa,KAAK,OAAO,MAAM;AAAA,IACtC;AACA,SAAK,aAAa,KAAK,OAAO,KAAK,6BAA6B,CAAC,CAAC;AAClE,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,SAAwB;AACtB,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAC5C,YAAM,UAAkC;AAAA,QACtC,cAAc;AAAA,MAChB;AACA,UAAI,KAAK,MAAM,SAAS;AAItB,YAAI,KAAK,MAAM,YAAY;AACzB,kBAAQ,gBAAgB,UAAU,KAAK,MAAM,UAAU;AAAA,QACzD,WAAW,KAAK,MAAM,QAAQ;AAC5B,kBAAQ,SAAS,IAAI,KAAK,MAAM;AAAA,QAClC,OAAO;AACL,iBAAO,IAAI,MAAM,6CAA6C,CAAC;AAC/D;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,gBAAgB,UAAU,KAAK,MAAM,MAAM;AACnD,gBAAQ,aAAa,IAAI;AAAA,MAC3B;AACA,YAAM,MAAM,IAAI,IAAI,CAAC,KAAK,MAAM,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAC9D,UAAI,IAAI,aAAa,UAAU;AAC7B,YAAI,WAAW;AAAA,MACjB;AAGA,YAAM,cAAsC,CAAC;AAC7C,UAAI,KAAK,MAAM,SAAS;AACtB,oBAAY,aAAa,IAAI,KAAK,MAAM,cAAc;AACtD,oBAAY,YAAY,IAAI,KAAK,MAAM;AAAA,MACzC,OAAO;AACL,oBAAY,OAAO,IAAI,KAAK,MAAM;AAAA,MACpC;AAEA,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,YAAI,aAAa,IAAI,KAAK,KAAK;AAAA,MACjC;AAEA,cAAQ,MAAM,yCAAyC,IAAI,SAAS,CAAC;AACrE,WAAK,MAAM,IAAI,oBAAU,IAAI,SAAS,GAAG;AAAA,QACvC;AAAA,MACF,CAAC;AAED,WAAK,IAAI,UAAU,CAAC,UAAU;AAC5B,eAAO,IAAI,MAAM,sCAAsC,MAAM,OAAO,CAAC;AAAA,MACvE;AAEA,gBAAM,yBAAK,KAAK,KAAK,MAAM;AAC3B,WAAK,WAAW;AAEhB,WAAK,IAAI,YAAY,CAAC,YAAY;AAChC,cAAM,QAA+B,KAAK,MAAM,QAAQ,IAAc;AACtE,aAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AACrE,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,aAAa,KAAK;AACvB;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,2BAA2B,KAAK;AACrC;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,wDAAwD,KAAK;AAClE;AAAA,UACF,KAAK;AACH,iBAAK,qDAAqD,KAAK;AAC/D;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,uBAAuB,KAAK;AACjC;AAAA,UACF,KAAK;AACH,iBAAK,oBAAoB,KAAK;AAC9B;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,8BAA8B,KAAK;AACxC;AAAA,UACF,KAAK;AACH,iBAAK,gCAAgC,KAAK;AAC1C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,wBAAwB,KAAK;AAClC;AAAA,UACF,KAAK;AACH,iBAAK,oCAAoC,KAAK;AAC9C;AAAA,UACF,KAAK;AACH,iBAAK,mCAAmC,KAAK;AAC7C;AAAA,UACF,KAAK;AACH,iBAAK,0BAA0B,KAAK;AACpC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,0CAA0C,KAAK;AACpD;AAAA,UACF,KAAK;AACH,iBAAK,yCAAyC,KAAK;AACnD;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,QACJ;AAAA,MACF;AAEA,YAAM,WAAW,YAAY;AAC3B,eAAO,KAAK,OAAO,CAAC,KAAK,YAAY,KAAK,IAAI,eAAe,oBAAU,MAAM;AAC3E,cAAI;AACF,kBAAM,QAAQ,MAAM,KAAK,WAAW,IAAI;AACxC,gBAAI,MAAM,SAAS,6BAA6B;AAC9C,mBAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AAAA,YACvE;AACA,iBAAK,IAAI,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,UACrC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,wBAAwB,KAAK;AAAA,UAClD;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAET,WAAK,IAAI,UAAU,MAAM;AACvB,YAAI,KAAK,cAAc,KAAK,IAAI,KAAK,KAAK,aAAa,KAAM;AAC3D,eAAK,WAAW;AAAA,QAClB;AACA,YAAI,CAAC,KAAK,UAAU;AAClB,iBAAO,IAAI,MAAM,gDAAgD,CAAC;AAAA,QACpE;AACA,aAAK,MAAM;AACX,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,CAAC,KAAK,IAAK;AACf,SAAK,WAAW;AAChB,SAAK,IAAI,MAAM;AACf,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,YAAY,KAAkC;AAC5C,UAAM,WAAW,KAAK,kBAAkB,IAAI,WAAW;AACvD,UAAM,SAAS,SAAU,OAAO,IAAI,YAAY;AAChD,UAAM,UAAU,OAAQ,QAAQ,IAAI,aAAa;AACjD,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmC;AAC9C,SAAK,QAAQ,MAAM,yBAAyB,KAAK,UAAU,MAAM,KAAK,CAAC,EAAE;AAAA,EAC3E;AAAA,EAEA,sBAAsB,OAA4C;AAChE,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,UAAU,KAAK,QAAQ,MAAM,EAAE,WAAW,KAAK,WAAW,CAAC;AAAA,EAClE;AAAA;AAAA,EAGA,sBAAsB,OAA4C;AAAA,EAAC;AAAA;AAAA,EAGnE,2BAA2B,OAAiD;AAAA,EAAC;AAAA,EAE7E,iCAAiC,OAAuD;AACtF,SAAK,KAAK,0BAA0B;AAAA,MAClC,QAAQ,MAAM;AAAA,IAChB,CAAyB;AAAA,EAC3B;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,qCAEE,OACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,QAAQ,MAAM;AAAA,IAChB,CAAuB;AAAA,EACzB;AAAA,EAEA,qCAEE,OACM;AACN,SAAK,KAAK,sBAAsB;AAAA,EAClC;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,wDACE,OACM;AACN,UAAM,aAAa,MAAM;AACzB,SAAK,KAAK,wCAAwC;AAAA,MAChD,QAAQ,MAAM;AAAA,MACd;AAAA,IACF,CAAsC;AAAA,EACxC;AAAA,EAEA,qDACE,OACM;AACN,UAAM,QAAQ,MAAM;AACpB,SAAK,QAAQ,MAAM,qDAAqD,MAAM,OAAO,EAAE;AACvF,SAAK,KAAK,qCAAqC;AAAA,MAC7C,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM;AAAA,IACjB,CAAmC;AAAA,EACrC;AAAA;AAAA,EAGA,iCAAiC,OAAuD;AAAA,EAAC;AAAA;AAAA,EAGzF,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,uBAAuB,iBAAuD;AAC5E,UAAM,WAAW,gBAAgB;AACjC,UAAM,UAAU,IAAI,qBAAO;AAC3B,UAAM,cAAgC;AAAA,MACpC,IAAI,SAAS;AAAA,MACb,QAAQ,SAAS;AAAA,MACjB,eAAe,SAAS;AAAA,MACxB,OAAO;AAAA,MACP,QAAQ,CAAC;AAAA,MACT;AAAA,MACA,kBAAkB,KAAK,IAAI;AAAA,IAC7B;AACA,SAAK,kBAAkB,YAAY,EAAE,IAAI;AACzC,SAAK,KAAK,oBAAoB,WAAW;AAAA,EAC3C;AAAA,EAEA,oBAAoB,OAA0C;AAC5D,UAAM,eAAe,MAAM;AAC3B,UAAM,aAAa,aAAa;AAChC,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,aAAS,SAAS,aAAa;AAC/B,aAAS,gBAAgB,aAAa;AACtC,aAAS,QAAQ,aAAa,SAAS;AACvC,SAAK,kBAAkB,UAAU,IAAI;AACrC,aAAS,QAAQ,QAAQ;AAEzB,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,SAAS,QAAQ;AAAA,MACvB,KAAK,UAAU;AACb,YAAI,SAAS,cAAc,SAAS,SAAU;AAC9C,cAAM,MAAM,SAAS,cAAc;AACnC,uBAAe,IAAI,sBAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B,MAAM,2BAAK;AAAA,UACX,SAAS,2BAAK;AAAA,QAChB,CAAC;AACD,aAAK,QACF,MAAM,EAAE,MAAM,2BAAK,MAAM,OAAO,2BAAK,QAAQ,CAAC,EAC9C,MAAM,4BAA4B;AACrC;AAAA,MACF;AAAA,MACA,KAAK,cAAc;AACjB,YAAI,SAAS,cAAc,SAAS,aAAc;AAClD,cAAM,SAAS,SAAS,cAAc;AACtC,uBAAe,IAAI,sBAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B;AAAA,QACF,CAAC;AACD,aAAK,QAAQ,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,gCAAgC;AACrE;AAAA,MACF;AAAA,MACA,KAAK,aAAa;AAChB,oBAAY;AACZ;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAK,iBAAiB,QAAQ;AAEnC,QAAI;AACJ,QAAI,SAAS,qBAAqB;AAChC,aAAO,SAAS,sBAAsB,SAAS;AAAA,IACjD;AACA,UAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AAEvC,UAAM,QAAQ,SAAS;AACvB,UAAM,SAAuC;AAAA,MAC3C,WAAW,SAAS;AAAA,MACpB,WAAW,SAAS;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,YAAY;AAAA,MACxB,mBAAkB,+BAAO,kBAAiB;AAAA,MAC1C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,mBAAmB,+BAAO,kBAAiB,KAAK,WAAY;AAAA,MAC5D,OAAO;AAAA,MACP,mBAAmB;AAAA,QACjB,eAAc,+BAAO,oBAAoB,kBAAiB;AAAA,QAC1D,aAAY,+BAAO,oBAAoB,gBAAe;AAAA,QACtD,cAAa,+BAAO,oBAAoB,iBAAgB;AAAA,MAC1D;AAAA,MACA,oBAAoB;AAAA,QAClB,aAAY,+BAAO,qBAAqB,gBAAe;AAAA,QACvD,cAAa,+BAAO,qBAAqB,iBAAgB;AAAA,MAC3D;AAAA,IACF;AACA,SAAK,KAAK,qBAAqB,MAAM;AAAA,EACvC;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,WAAW,MAAM;AAEvB,QAAI,SAAS,SAAS,aAAa,SAAS,SAAS,iBAAiB;AACpE,YAAM,IAAI,MAAM,yBAAyB,SAAS,IAAI,EAAE;AAAA,IAC1D;AAEA,QAAI;AACJ,QAAI,SAAS,SAAS,iBAAiB;AACrC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,SAAS;AAAA,IAClB;AAEA,UAAM,YAA4B;AAAA,MAChC;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,aAAa,MAAM;AAAA,MACnB,MAAM,SAAS;AAAA,MACf;AAAA,MACA,SAAS,CAAC;AAAA,MACV,SAAS,IAAI,qBAAO;AAAA,IACtB;AACA,yCAAU,OAAO,KAAK;AACtB,SAAK,KAAK,yBAAyB,SAAS;AAAA,EAC9C;AAAA,EAEA,8BAA8B,OAAoD;AAChF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,SAAI,iCAAQ,UAAS,iBAAiB;AACpC,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,QAAQ,MAAM,mDAAmD;AACtE;AAAA,MACF;AAGA,YAAM,OAAO,MAAM;AACnB,UAAI,KAAK,SAAS,iBAAiB;AACjC,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,YAAM,OAAO,KAAK,QAAQ,KAAK,IAAI;AACnC,UAAI,CAAC,MAAM;AACT,aAAK,QAAQ,MAAM,yBAAyB,KAAK,IAAI,YAAY;AACjE;AAAA,MACF;AAEA,WAAK,KAAK,yBAAyB;AAAA,QACjC,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,aAAa,KAAK,MAAM,KAAK,SAAS;AAE5C,WAAK,QAAQ;AAAA,QACX,kBAAkB,KAAK,OAAO,eAAe,KAAK,IAAI,mBAAmB,UAAU;AAAA,MACrF;AAEA,WAAK,QAAQ,UAAU,EAAE;AAAA,QACvB,CAAC,YAAY;AACX,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,aAAa,OAAO,EAAE;AACrF,eAAK,KAAK,2BAA2B;AAAA,YACnC,QAAQ,KAAK;AAAA,UACf,CAAC;AACD,eAAK,aAAa,KAAK;AAAA,YACrB,kBAAI,YAAY,6BAA6B;AAAA,cAC3C,MAAM,KAAK;AAAA,cACX,YAAY,KAAK;AAAA,cACjB,QAAQ;AAAA,YACV,CAAC;AAAA,YACD,OAAO;AAAA,UACT;AACA,eAAK,SAAS,OAAO;AAAA,QACvB;AAAA,QACA,CAAC,UAAU;AACT,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,gBAAgB,KAAK,EAAE;AAEtF,eAAK,KAAK,wBAAwB;AAAA,YAChC,QAAQ,KAAK;AAAA,UACf,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,qCAAQ,QAAQ;AAChB,SAAK,KAAK,wBAAwB,MAAM;AAAA,EAC1C;AAAA,EAEA,gCAAgC,OAAsD;AACpF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,UAAM,aAAa,IAAI,iCAA2B;AAClD,UAAM,cAAc,IAAI,iCAA+B;AAEvD,UAAM,aAA8B;AAAA,MAClC;AAAA,MACA,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,cAAc,MAAM;AAAA,MACpB,MAAM;AAAA,MACN,OAAO,CAAC;AAAA,MACR;AAAA,MACA;AAAA,MACA,WAAW,CAAC;AAAA,MACZ,aAAa,MAAM,KAAK;AAAA,IAC1B;AACA,qCAAQ,QAAQ,KAAK;AACrB,aAAU,sBAAsB,KAAK,IAAI;AACzC,SAAK,KAAK,0BAA0B,UAAU;AAAA,EAChD;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,SAAK,KAAK,yBAAyB,OAAO;AAAA,EAC5C;AAAA,EAEA,yBAAyB,OAA+C;AACtE,SAAK,KAAK,uBAAuB,KAAK;AAAA,EACxC;AAAA,EAEA,wBAAwB,OAA8C;AACpE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,OAAO,MAAM;AACrB,SAAK,KAAK,sBAAsB,KAAK;AAAA,EACvC;AAAA,EAEA,oCAAoC,OAA0D;AAC5F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,aAAa,MAAM;AACzB,YAAQ,QAAQ;AAEhB,YAAQ,WAAW,IAAI,UAAU;AAAA,EACnC;AAAA,EAEA,mCAAmC,OAAyD;AAC1F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,WAAW,MAAM;AAAA,EAC3B;AAAA,EAEA,0BAA0B,OAAgD;AACxE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO,QAAQ;AAC9C,UAAM,QAAQ,IAAI;AAAA,MAChB,IAAI,WAAW,KAAK,MAAM;AAAA,MAC1B,UAAU;AAAA,MACV,UAAU;AAAA,MACV,KAAK,SAAS;AAAA,IAChB;AACA,YAAQ,MAAM,KAAK,KAAK;AAExB,YAAQ,YAAY,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,yBAAyB,OAA+C;AACtE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,YAAY,MAAM;AAAA,EAC5B;AAAA,EAEA,0CAEE,OACM;AAAA,EAAC;AAAA,EAET,yCAEE,OACM;AAAA,EAAC;AAAA;AAAA,EAGT,yBAAyB,OAA+C;AAAA,EAAC;AAC3E;","names":["c"]}

package/dist/realtime/realtime_model.d.ts

@@ -162,7 +162,7 @@ export declare class RealtimeSession extends multimodal.RealtimeSession {
     get response(): Response;
     get expiration(): number;
     queueMsg(command: api_proto.ClientEvent): void;
-    sessionUpdate({ modalities, instructions, voice, inputAudioFormat, outputAudioFormat, inputAudioTranscription, turnDetection, temperature, maxResponseOutputTokens, toolChoice, }: {
+    sessionUpdate({ modalities, instructions, voice, inputAudioFormat, outputAudioFormat, inputAudioTranscription, turnDetection, temperature, maxResponseOutputTokens, toolChoice, selectedTools, }: {
         modalities: ['text', 'audio'] | ['text'];
         instructions?: string;
         voice?: api_proto.Voice;
@@ -173,6 +173,7 @@ export declare class RealtimeSession extends multimodal.RealtimeSession {
         temperature?: number;
         maxResponseOutputTokens?: number;
         toolChoice?: api_proto.ToolChoice;
+        selectedTools?: string[];
     }): void;
     /**
      * Try to recover from a text response to audio mode.

package/dist/realtime/realtime_model.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"realtime_model.d.ts","sourceRoot":"","sources":["../../src/realtime/realtime_model.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,kBAAkB,EAClB,MAAM,EAEN,GAAG,EAIH,UAAU,EACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAC;AAE5C,UAAU,YAAY;IACpB,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC;IACvB,gBAAgB,EAAE,SAAS,CAAC,WAAW,CAAC;IACxC,iBAAiB,EAAE,SAAS,CAAC,WAAW,CAAC;IACzC,uBAAuB,EAAE,SAAS,CAAC,uBAAuB,GAAG,IAAI,CAAC;IAClE,aAAa,EAAE,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,CAAC;IAChC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,CAAC,cAAc,CAAC;IACjC,aAAa,EAAE,SAAS,CAAC,qBAAqB,GAAG,IAAI,CAAC;IACtD,KAAK,EAAE,SAAS,CAAC,UAAU,GAAG,IAAI,CAAC;IACnC,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC;IACrB,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAClC,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,UAAU,EAAE,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACvC,WAAW,EAAE,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAC5C,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iCAAiC;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,cAAM,gBAAgB;;gBAGR,OAAO,EAAE,eAAe;IAIpC,MAAM,CAAC,KAAK,EAAE,UAAU;IAOxB,KAAK;IAML,MAAM;CAKP;AAED,cAAM,gBAAgB;;gBAIR,OAAO,EAAE,eAAe;IAIpC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;IAS/D,MAAM,CAAC,MAAM,EAAE,MAAM;IAOrB,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;CAyHhE;AAED,cAAM,YAAY;;gBAGJ,OAAO,EAAE,eAAe;IAIpC,IAAI,IAAI,IAAI,gBAAgB,CAE3B;CACF;AAED,cAAM,QAAQ;;gBAGA,OAAO,EAAE,eAAe;IAIpC,MAAM;IAMN,MAAM;CAKP;AAQD,qBAAa,aAAc,SAAQ,UAAU,CAAC,aAAa;;IACzD,UAAU,SAAyB;IACnC,WAAW,SAA0B;IACrC,WAAW,SAA2B;IACtC,YAAY,SAA4B;IAKxC,MAAM,CAAC,SAAS,CAAC,EACf,OAAO,EACP,eAAe,EACf,UAAiC,EACjC,MAAkB,EAClB,UAAsB,EACtB,YAAiB,EACjB,UAA8B,EAC9B,KAAe,EACf,gBAA0B,EAC1B,iBAA2B,EAC3B,uBAAgD,EAChD,aAAsC,EACtC,WAAiB,EACjB,uBAAkC,GACnC,EAAE;QACD,OAAO,EAAE,MAAM,CAAC;QAChB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,CAAC;QAC5D,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;QAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;KAClC;gBAoBW,EACV,UAA8B,EAC9B,YAAiB,EACjB,KAAe,EACf,gBAA0B,EAC1B,iBAA2B,EAC3B,uBAAgD,EAChD,aAAsC,EACtC,WAAiB,EACjB,uBAAkC,EAClC,KAA4C,EAC5C,MAAyC,EACzC,OAA4B,EAE5B,OAAe,EACf,UAAsB,EACtB,UAAsB,GACvB,EAAE;QACD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,CAAC;QAC5D,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;QAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IA4BD,IAAI,QAAQ,IAAI,eAAe,EAAE,CAEhC;IAED,OAAO,CAAC,EACN,MAAM,EACN,OAAO,EACP,UAAyC,EACzC,YAA6C,EAC7C,KAA+B,EAC/B,gBAAqD,EACrD,iBAAuD,EACvD,uBAAmE,EACnE,aAA+C,EAC/C,WAA2C,EAC3C,uBAAmE,GACpE,EAAE;QACD,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;QAC7B,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACnE,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACnD,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;KAClC,GAAG,eAAe;IA2Bb,KAAK;CAGZ;AAED,qBAAa,eAAgB,SAAQ,UAAU,CAAC,eAAe;;gBAc3D,IAAI,EAAE,YAAY,EAClB,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAAE,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;QAAC,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAA;KAAE;IAwBlF,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,GAAG,SAAS,CAEzC;IAED,IAAI,MAAM,IAAI,GAAG,CAAC,eAAe,GAAG,SAAS,CAE5C;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EAE9C;IAED,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED,IAAI,gBAAgB,IAAI,gBAAgB,CAEvC;IAED,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,UAAU,IAAI,MAAM,CAKvB;IAED,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,WAAW,GAAG,IAAI;IAkC9C,aAAa,CAAC,EACZ,UAAkC,EAClC,YAAsC,EACtC,KAAwB,EACxB,gBAA8C,EAC9C,iBAAgD,EAChD,uBAA4D,EAC5D,aAAwC,EACxC,WAAoC,EACpC,uBAA4D,EAC5D,UAAmB,GACpB,EAAE;QACD,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACnE,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACnD,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,UAAU,CAAC,EAAE,SAAS,CAAC,UAAU,CAAC;KACnC;IA4ED;;;;;;;OAOG;IACH,uBAAuB,CAAC,MAAM,EAAE,MAAM;IAmLhC,KAAK;CAiWZ"}
+{"version":3,"file":"realtime_model.d.ts","sourceRoot":"","sources":["../../src/realtime/realtime_model.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,kBAAkB,EAClB,MAAM,EAEN,GAAG,EAIH,UAAU,EACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAC;AAE5C,UAAU,YAAY;IACpB,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC;IACvB,gBAAgB,EAAE,SAAS,CAAC,WAAW,CAAC;IACxC,iBAAiB,EAAE,SAAS,CAAC,WAAW,CAAC;IACzC,uBAAuB,EAAE,SAAS,CAAC,uBAAuB,GAAG,IAAI,CAAC;IAClE,aAAa,EAAE,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,CAAC;IAChC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,CAAC,cAAc,CAAC;IACjC,aAAa,EAAE,SAAS,CAAC,qBAAqB,GAAG,IAAI,CAAC;IACtD,KAAK,EAAE,SAAS,CAAC,UAAU,GAAG,IAAI,CAAC;IACnC,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC;IACrB,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAClC,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,UAAU,EAAE,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACvC,WAAW,EAAE,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAC5C,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iCAAiC;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,cAAM,gBAAgB;;gBAGR,OAAO,EAAE,eAAe;IAIpC,MAAM,CAAC,KAAK,EAAE,UAAU;IAOxB,KAAK;IAML,MAAM;CAKP;AAED,cAAM,gBAAgB;;gBAIR,OAAO,EAAE,eAAe;IAIpC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;IAS/D,MAAM,CAAC,MAAM,EAAE,MAAM;IAOrB,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;CAyHhE;AAED,cAAM,YAAY;;gBAGJ,OAAO,EAAE,eAAe;IAIpC,IAAI,IAAI,IAAI,gBAAgB,CAE3B;CACF;AAED,cAAM,QAAQ;;gBAGA,OAAO,EAAE,eAAe;IAIpC,MAAM;IAMN,MAAM;CAKP;AAQD,qBAAa,aAAc,SAAQ,UAAU,CAAC,aAAa;;IACzD,UAAU,SAAyB;IACnC,WAAW,SAA0B;IACrC,WAAW,SAA2B;IACtC,YAAY,SAA4B;IAKxC,MAAM,CAAC,SAAS,CAAC,EACf,OAAO,EACP,eAAe,EACf,UAAiC,EACjC,MAAkB,EAClB,UAAsB,EACtB,YAAiB,EACjB,UAA8B,EAC9B,KAAe,EACf,gBAA0B,EAC1B,iBAA2B,EAC3B,uBAAgD,EAChD,aAAsC,EACtC,WAAiB,EACjB,uBAAkC,GACnC,EAAE;QACD,OAAO,EAAE,MAAM,CAAC;QAChB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,CAAC;QAC5D,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;QAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;KAClC;gBAoBW,EACV,UAA8B,EAC9B,YAAiB,EACjB,KAAe,EACf,gBAA0B,EAC1B,iBAA2B,EAC3B,uBAAgD,EAChD,aAAsC,EACtC,WAAiB,EACjB,uBAAkC,EAClC,KAA4C,EAC5C,MAAyC,EACzC,OAA4B,EAE5B,OAAe,EACf,UAAsB,EACtB,UAAsB,GACvB,EAAE;QACD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,CAAC;QAC5D,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,CAAC;QAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IA4BD,IAAI,QAAQ,IAAI,eAAe,EAAE,CAEhC;IAED,OAAO,CAAC,EACN,MAAM,EACN,OAAO,EACP,UAAyC,EACzC,YAA6C,EAC7C,KAA+B,EAC/B,gBAAqD,EACrD,iBAAuD,EACvD,uBAAmE,EACnE,aAA+C,EAC/C,WAA2C,EAC3C,uBAAmE,GACpE,EAAE;QACD,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;QAC7B,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACnE,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACnD,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;KAClC,GAAG,eAAe;IA2Bb,KAAK;CAGZ;AAED,qBAAa,eAAgB,SAAQ,UAAU,CAAC,eAAe;;gBAc3D,IAAI,EAAE,YAAY,EAClB,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE;QAAE,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;QAAC,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAA;KAAE;IAwBlF,IAAI,OAAO,IAAI,GAAG,CAAC,WAAW,GAAG,SAAS,CAEzC;IAED,IAAI,MAAM,IAAI,GAAG,CAAC,eAAe,GAAG,SAAS,CAE5C;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EAE9C;IAED,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED,IAAI,gBAAgB,IAAI,gBAAgB,CAEvC;IAED,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,UAAU,IAAI,MAAM,CAKvB;IAED,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,WAAW,GAAG,IAAI;IAkC9C,aAAa,CAAC,EACZ,UAAkC,EAClC,YAAsC,EACtC,KAAwB,EACxB,gBAA8C,EAC9C,iBAAgD,EAChD,uBAA4D,EAC5D,aAAwC,EACxC,WAAoC,EACpC,uBAA4D,EAC5D,UAAmB,EACnB,aAA+C,GAChD,EAAE;QACD,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC;QACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QACzC,iBAAiB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC;QAC1C,uBAAuB,CAAC,EAAE,SAAS,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACnE,aAAa,CAAC,EAAE,SAAS,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACnD,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;QACjC,UAAU,CAAC,EAAE,SAAS,CAAC,UAAU,CAAC;QAClC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;KAC1B;IA8ED;;;;;;;OAOG;IACH,uBAAuB,CAAC,MAAM,EAAE,MAAM;IAmLhC,KAAK;CAiWZ"}

package/dist/realtime/realtime_model.js

@@ -405,7 +405,8 @@ class RealtimeSession extends multimodal.RealtimeSession {
     turnDetection = this.#opts.turnDetection,
     temperature = this.#opts.temperature,
     maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,
-    toolChoice = "auto"
+    toolChoice = "auto",
+    selectedTools = Object.keys(this.#fncCtx || {})
   }) {
     this.#opts = {
       modalities,
@@ -424,7 +425,7 @@ class RealtimeSession extends multimodal.RealtimeSession {
       apiVersion: this.#opts.apiVersion,
       entraToken: this.#opts.entraToken
     };
-    const tools = this.#fncCtx ? Object.entries(this.#fncCtx).map(([name, func]) => ({
+    const tools = this.#fncCtx ? Object.entries(this.#fncCtx).filter(([name]) => selectedTools.includes(name)).map(([name, func]) => ({
       type: "function",
       name,
       description: func.description,
@@ -508,7 +509,7 @@ class RealtimeSession extends multimodal.RealtimeSession {
       }
       const queryParams = {};
       if (this.#opts.isAzure) {
-        queryParams["api-version"] = "2024-10-01-preview";
+        queryParams["api-version"] = this.#opts.apiVersion ?? "2024-10-01-preview";
         queryParams["deployment"] = this.#opts.model;
       } else {
         queryParams["model"] = this.#opts.model;

package/dist/realtime/realtime_model.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AsyncIterableQueue,\n  Future,\n  Queue,\n  llm,\n  log,\n  mergeFrames,\n  metrics,\n  multimodal,\n} from '@livekit/agents';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { once } from 'node:events';\nimport { WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\ninterface ModelOptions {\n  modalities: ['text', 'audio'] | ['text'];\n  instructions: string;\n  voice: api_proto.Voice;\n  inputAudioFormat: api_proto.AudioFormat;\n  outputAudioFormat: api_proto.AudioFormat;\n  inputAudioTranscription: api_proto.InputAudioTranscription | null;\n  turnDetection: api_proto.TurnDetectionType | null;\n  temperature: number;\n  maxResponseOutputTokens: number;\n  model: api_proto.Model;\n  apiKey?: string;\n  baseURL: string;\n  isAzure: boolean;\n  entraToken?: string;\n  apiVersion?: string;\n}\n\nexport interface RealtimeResponse {\n  id: string;\n  status: api_proto.ResponseStatus;\n  statusDetails: api_proto.ResponseStatusDetails | null;\n  usage: api_proto.ModelUsage | null;\n  output: RealtimeOutput[];\n  doneFut: Future;\n  createdTimestamp: number;\n  firstTokenTimestamp?: number;\n}\n\nexport interface RealtimeOutput {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  role: api_proto.Role;\n  type: 'message' | 'function_call';\n  content: RealtimeContent[];\n  doneFut: Future;\n}\n\nexport interface RealtimeContent {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  contentIndex: number;\n  text: string;\n  audio: AudioFrame[];\n  textStream: AsyncIterableQueue<string>;\n  audioStream: AsyncIterableQueue<AudioFrame>;\n  toolCalls: RealtimeToolCall[];\n  contentType: api_proto.Modality;\n}\n\nexport interface RealtimeToolCall {\n  name: string;\n  arguments: string;\n  toolCallID: string;\n}\n\nexport interface InputSpeechTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n}\n\nexport interface InputSpeechTranscriptionFailed {\n  itemId: string;\n  message: string;\n}\n\nexport interface InputSpeechStarted {\n  itemId: string;\n}\n\nexport interface InputSpeechCommitted {\n  itemId: string;\n}\n\nclass InputAudioBuffer {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  append(frame: AudioFrame) {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.append',\n      audio: Buffer.from(frame.data.buffer).toString('base64'),\n    });\n  }\n\n  clear() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.clear',\n    });\n  }\n\n  commit() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.commit',\n    });\n  }\n}\n\nclass ConversationItem {\n  #session: RealtimeSession;\n  #logger = log();\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  truncate(itemId: string, contentIndex: number, audioEnd: number) {\n    this.#session.queueMsg({\n      type: 'conversation.item.truncate',\n      item_id: itemId,\n      content_index: contentIndex,\n      audio_end_ms: audioEnd,\n    });\n  }\n\n  delete(itemId: string) {\n    this.#session.queueMsg({\n      type: 'conversation.item.delete',\n      item_id: itemId,\n    });\n  }\n\n  create(message: llm.ChatMessage, previousItemId?: string): void {\n    if (!message.content) {\n      return;\n    }\n\n    let event: api_proto.ConversationItemCreateEvent;\n\n    if (message.toolCallId) {\n      if (typeof message.content !== 'string') {\n        throw new TypeError('message.content must be a string');\n      }\n\n      event = {\n        type: 'conversation.item.create',\n        previous_item_id: previousItemId,\n        item: {\n          type: 'function_call_output',\n          call_id: message.toolCallId,\n          output: message.content,\n        },\n      };\n    } else {\n      let content = message.content;\n      if (!Array.isArray(content)) {\n        content = [content];\n      }\n\n      if (message.role === llm.ChatRole.USER) {\n        const contents: (api_proto.InputTextContent | api_proto.InputAudioContent)[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            contents.push({\n              type: 'input_audio',\n              audio: Buffer.from(mergeFrames(c.frame).data.buffer).toString('base64'),\n            });\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'user',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.ASSISTANT) {\n        const contents: api_proto.TextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in assistant message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'assistant',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.SYSTEM) {\n        const contents: api_proto.InputTextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in system message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'system',\n            content: contents,\n          },\n        };\n      } else {\n        this.#logger\n          .child({ message })\n          .warn('chat message is not supported inside the realtime API');\n        return;\n      }\n    }\n\n    this.#session.queueMsg(event);\n  }\n}\n\nclass Conversation {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  get item(): ConversationItem {\n    return new ConversationItem(this.#session);\n  }\n}\n\nclass Response {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  create() {\n    this.#session.queueMsg({\n      type: 'response.create',\n    });\n  }\n\n  cancel() {\n    this.#session.queueMsg({\n      type: 'response.cancel',\n    });\n  }\n}\n\ninterface ContentPtr {\n  response_id: string;\n  output_index: number;\n  content_index: number;\n}\n\nexport class RealtimeModel extends multimodal.RealtimeModel {\n  sampleRate = api_proto.SAMPLE_RATE;\n  numChannels = api_proto.NUM_CHANNELS;\n  inFrameSize = api_proto.IN_FRAME_SIZE;\n  outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n  #defaultOpts: ModelOptions;\n  #sessions: RealtimeSession[] = [];\n\n  static withAzure({\n    baseURL,\n    azureDeployment,\n    apiVersion = '2024-10-01-preview',\n    apiKey = undefined,\n    entraToken = undefined,\n    instructions = '',\n    modalities = ['text', 'audio'],\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n  }: {\n    baseURL: string;\n    azureDeployment: string;\n    apiVersion?: string;\n    apiKey?: string;\n    entraToken?: string;\n    instructions?: string;\n    modalities?: ['text', 'audio'] | ['text'];\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }) {\n    return new RealtimeModel({\n      isAzure: true,\n      baseURL: new URL('openai', baseURL).toString(),\n      model: azureDeployment,\n      apiVersion,\n      apiKey,\n      entraToken,\n      instructions,\n      modalities,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n    });\n  }\n\n  constructor({\n    modalities = ['text', 'audio'],\n    instructions = '',\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n    model = 'gpt-4o-realtime-preview-2024-10-01',\n    apiKey = process.env.OPENAI_API_KEY || '',\n    baseURL = api_proto.BASE_URL,\n    // used for microsoft\n    isAzure = false,\n    apiVersion = undefined,\n    entraToken = undefined,\n  }: {\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    model?: api_proto.Model;\n    apiKey?: string;\n    baseURL?: string;\n    isAzure?: boolean;\n    apiVersion?: string;\n    entraToken?: string;\n  }) {\n    super();\n\n    if (apiKey === '') {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable',\n      );\n    }\n\n    this.#defaultOpts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model,\n      apiKey,\n      baseURL,\n      isAzure,\n      apiVersion,\n      entraToken,\n    };\n  }\n\n  get sessions(): RealtimeSession[] {\n    return this.#sessions;\n  }\n\n  session({\n    fncCtx,\n    chatCtx,\n    modalities = this.#defaultOpts.modalities,\n    instructions = this.#defaultOpts.instructions,\n    voice = this.#defaultOpts.voice,\n    inputAudioFormat = this.#defaultOpts.inputAudioFormat,\n    outputAudioFormat = this.#defaultOpts.outputAudioFormat,\n    inputAudioTranscription = this.#defaultOpts.inputAudioTranscription,\n    turnDetection = this.#defaultOpts.turnDetection,\n    temperature = this.#defaultOpts.temperature,\n    maxResponseOutputTokens = this.#defaultOpts.maxResponseOutputTokens,\n  }: {\n    fncCtx?: llm.FunctionContext;\n    chatCtx?: llm.ChatContext;\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }): RealtimeSession {\n    const opts: ModelOptions = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#defaultOpts.model,\n      apiKey: this.#defaultOpts.apiKey,\n      baseURL: this.#defaultOpts.baseURL,\n      isAzure: this.#defaultOpts.isAzure,\n      apiVersion: this.#defaultOpts.apiVersion,\n      entraToken: this.#defaultOpts.entraToken,\n    };\n\n    const newSession = new RealtimeSession(opts, {\n      chatCtx: chatCtx || new llm.ChatContext(),\n      fncCtx,\n    });\n    this.#sessions.push(newSession);\n    return newSession;\n  }\n\n  async close() {\n    await Promise.allSettled(this.#sessions.map((session) => session.close()));\n  }\n}\n\nexport class RealtimeSession extends multimodal.RealtimeSession {\n  #chatCtx: llm.ChatContext | undefined = undefined;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #opts: ModelOptions;\n  #pendingResponses: { [id: string]: RealtimeResponse } = {};\n  #sessionId = 'not-connected';\n  #ws: WebSocket | null = null;\n  #expiresAt: number | null = null;\n  #logger = log();\n  #task: Promise<void>;\n  #closing = true;\n  #sendQueue = new Queue<api_proto.ClientEvent>();\n\n  constructor(\n    opts: ModelOptions,\n    { fncCtx, chatCtx }: { fncCtx?: llm.FunctionContext; chatCtx?: llm.ChatContext },\n  ) {\n    super();\n\n    this.#opts = opts;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n\n    this.#task = this.#start();\n\n    this.sessionUpdate({\n      modalities: this.#opts.modalities,\n      instructions: this.#opts.instructions,\n      voice: this.#opts.voice,\n      inputAudioFormat: this.#opts.inputAudioFormat,\n      outputAudioFormat: this.#opts.outputAudioFormat,\n      inputAudioTranscription: this.#opts.inputAudioTranscription,\n      turnDetection: this.#opts.turnDetection,\n      temperature: this.#opts.temperature,\n      maxResponseOutputTokens: this.#opts.maxResponseOutputTokens,\n      toolChoice: 'auto',\n    });\n  }\n\n  get chatCtx(): llm.ChatContext | undefined {\n    return this.#chatCtx;\n  }\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n  }\n\n  get conversation(): Conversation {\n    return new Conversation(this);\n  }\n\n  get inputAudioBuffer(): InputAudioBuffer {\n    return new InputAudioBuffer(this);\n  }\n\n  get response(): Response {\n    return new Response(this);\n  }\n\n  get expiration(): number {\n    if (!this.#expiresAt) {\n      throw new Error('session not started');\n    }\n    return this.#expiresAt * 1000;\n  }\n\n  queueMsg(command: api_proto.ClientEvent): void {\n    this.#sendQueue.put(command);\n  }\n\n  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs\n  /// with large amounts of base64 audio data.\n  #loggableEvent(\n    event: api_proto.ClientEvent | api_proto.ServerEvent,\n    maxLength: number = 30,\n  ): Record<string, unknown> {\n    const untypedEvent: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(event)) {\n      if (value !== undefined) {\n        untypedEvent[key] = value;\n      }\n    }\n\n    if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n      const truncatedData =\n        untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? '…' : '');\n      return { ...untypedEvent, audio: truncatedData };\n    }\n    if (\n      untypedEvent.delta &&\n      typeof untypedEvent.delta === 'string' &&\n      event.type === 'response.audio.delta'\n    ) {\n      const truncatedDelta =\n        untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? '…' : '');\n      return { ...untypedEvent, delta: truncatedDelta };\n    }\n    return untypedEvent;\n  }\n\n  sessionUpdate({\n    modalities = this.#opts.modalities,\n    instructions = this.#opts.instructions,\n    voice = this.#opts.voice,\n    inputAudioFormat = this.#opts.inputAudioFormat,\n    outputAudioFormat = this.#opts.outputAudioFormat,\n    inputAudioTranscription = this.#opts.inputAudioTranscription,\n    turnDetection = this.#opts.turnDetection,\n    temperature = this.#opts.temperature,\n    maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,\n    toolChoice = 'auto',\n  }: {\n    modalities: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    toolChoice?: api_proto.ToolChoice;\n  }) {\n    this.#opts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#opts.model,\n      apiKey: this.#opts.apiKey,\n      baseURL: this.#opts.baseURL,\n      isAzure: this.#opts.isAzure,\n      apiVersion: this.#opts.apiVersion,\n      entraToken: this.#opts.entraToken,\n    };\n\n    const tools = this.#fncCtx\n      ? Object.entries(this.#fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          name,\n          description: func.description,\n          parameters:\n            // don't format parameters if they are raw openai params\n            func.parameters.type == ('object' as const)\n              ? func.parameters\n              : llm.oaiParams(func.parameters),\n        }))\n      : [];\n\n    const sessionUpdateEvent: api_proto.SessionUpdateEvent = {\n      type: 'session.update',\n      session: {\n        modalities: this.#opts.modalities,\n        instructions: this.#opts.instructions,\n        voice: this.#opts.voice,\n        input_audio_format: this.#opts.inputAudioFormat,\n        output_audio_format: this.#opts.outputAudioFormat,\n        input_audio_transcription: this.#opts.inputAudioTranscription,\n        turn_detection: this.#opts.turnDetection,\n        temperature: this.#opts.temperature,\n        max_response_output_tokens:\n          this.#opts.maxResponseOutputTokens === Infinity\n            ? 'inf'\n            : this.#opts.maxResponseOutputTokens,\n        tools,\n        tool_choice: toolChoice,\n      },\n    };\n\n    if (this.#opts.isAzure && this.#opts.maxResponseOutputTokens === Infinity) {\n      // microsoft doesn't support inf for max_response_output_tokens, but accepts no args\n      sessionUpdateEvent.session.max_response_output_tokens = undefined;\n    }\n\n    this.queueMsg(sessionUpdateEvent);\n  }\n\n  /** Create an empty audio message with the given duration. */\n  #createEmptyUserAudioMessage(duration: number): llm.ChatMessage {\n    const samples = duration * api_proto.SAMPLE_RATE;\n    return new llm.ChatMessage({\n      role: llm.ChatRole.USER,\n      content: {\n        frame: new AudioFrame(\n          new Int16Array(samples * api_proto.NUM_CHANNELS),\n          api_proto.SAMPLE_RATE,\n          api_proto.NUM_CHANNELS,\n          samples,\n        ),\n      },\n    });\n  }\n\n  /**\n   * Try to recover from a text response to audio mode.\n   *\n   * @remarks\n   * Sometimes the OpenAI Realtime API returns text instead of audio responses.\n   * This method tries to recover from this by requesting a new response after deleting the text\n   * response and creating an empty user audio message.\n   */\n  recoverFromTextResponse(itemId: string) {\n    if (itemId) {\n      this.conversation.item.delete(itemId);\n    }\n    this.conversation.item.create(this.#createEmptyUserAudioMessage(1));\n    this.response.create();\n  }\n\n  #start(): Promise<void> {\n    return new Promise(async (resolve, reject) => {\n      const headers: Record<string, string> = {\n        'User-Agent': 'LiveKit-Agents-JS',\n      };\n      if (this.#opts.isAzure) {\n        // Microsoft API has two ways of authentication\n        // 1. Entra token set as `Bearer` token\n        // 2. API key set as `api_key` header (also accepts query string)\n        if (this.#opts.entraToken) {\n          headers.Authorization = `Bearer ${this.#opts.entraToken}`;\n        } else if (this.#opts.apiKey) {\n          headers['api-key'] = this.#opts.apiKey;\n        } else {\n          reject(new Error('Microsoft API key or entraToken is required'));\n          return;\n        }\n      } else {\n        headers.Authorization = `Bearer ${this.#opts.apiKey}`;\n        headers['OpenAI-Beta'] = 'realtime=v1';\n      }\n      const url = new URL([this.#opts.baseURL, 'realtime'].join('/'));\n      if (url.protocol === 'https:') {\n        url.protocol = 'wss:';\n      }\n\n      // Construct query parameters\n      const queryParams: Record<string, string> = {};\n      if (this.#opts.isAzure) {\n        queryParams['api-version'] = '2024-10-01-preview';\n        queryParams['deployment'] = this.#opts.model;\n      } else {\n        queryParams['model'] = this.#opts.model;\n      }\n\n      for (const [key, value] of Object.entries(queryParams)) {\n        url.searchParams.set(key, value);\n      }\n\n      console.debug('Connecting to OpenAI Realtime API at ', url.toString());\n      this.#ws = new WebSocket(url.toString(), {\n        headers: headers,\n      });\n\n      this.#ws.onerror = (error) => {\n        reject(new Error('OpenAI Realtime WebSocket error: ' + error.message));\n      };\n\n      await once(this.#ws, 'open');\n      this.#closing = false;\n\n      this.#ws.onmessage = (message) => {\n        const event: api_proto.ServerEvent = JSON.parse(message.data as string);\n        this.#logger.debug(`<- ${JSON.stringify(this.#loggableEvent(event))}`);\n        switch (event.type) {\n          case 'error':\n            this.#handleError(event);\n            break;\n          case 'session.created':\n            this.#handleSessionCreated(event);\n            break;\n          case 'session.updated':\n            this.#handleSessionUpdated(event);\n            break;\n          case 'conversation.created':\n            this.#handleConversationCreated(event);\n            break;\n          case 'input_audio_buffer.committed':\n            this.#handleInputAudioBufferCommitted(event);\n            break;\n          case 'input_audio_buffer.cleared':\n            this.#handleInputAudioBufferCleared(event);\n            break;\n          case 'input_audio_buffer.speech_started':\n            this.#handleInputAudioBufferSpeechStarted(event);\n            break;\n          case 'input_audio_buffer.speech_stopped':\n            this.#handleInputAudioBufferSpeechStopped(event);\n            break;\n          case 'conversation.item.created':\n            this.#handleConversationItemCreated(event);\n            break;\n          case 'conversation.item.input_audio_transcription.completed':\n            this.#handleConversationItemInputAudioTranscriptionCompleted(event);\n            break;\n          case 'conversation.item.input_audio_transcription.failed':\n            this.#handleConversationItemInputAudioTranscriptionFailed(event);\n            break;\n          case 'conversation.item.truncated':\n            this.#handleConversationItemTruncated(event);\n            break;\n          case 'conversation.item.deleted':\n            this.#handleConversationItemDeleted(event);\n            break;\n          case 'response.created':\n            this.#handleResponseCreated(event);\n            break;\n          case 'response.done':\n            this.#handleResponseDone(event);\n            break;\n          case 'response.output_item.added':\n            this.#handleResponseOutputItemAdded(event);\n            break;\n          case 'response.output_item.done':\n            this.#handleResponseOutputItemDone(event);\n            break;\n          case 'response.content_part.added':\n            this.#handleResponseContentPartAdded(event);\n            break;\n          case 'response.content_part.done':\n            this.#handleResponseContentPartDone(event);\n            break;\n          case 'response.text.delta':\n            this.#handleResponseTextDelta(event);\n            break;\n          case 'response.text.done':\n            this.#handleResponseTextDone(event);\n            break;\n          case 'response.audio_transcript.delta':\n            this.#handleResponseAudioTranscriptDelta(event);\n            break;\n          case 'response.audio_transcript.done':\n            this.#handleResponseAudioTranscriptDone(event);\n            break;\n          case 'response.audio.delta':\n            this.#handleResponseAudioDelta(event);\n            break;\n          case 'response.audio.done':\n            this.#handleResponseAudioDone(event);\n            break;\n          case 'response.function_call_arguments.delta':\n            this.#handleResponseFunctionCallArgumentsDelta(event);\n            break;\n          case 'response.function_call_arguments.done':\n            this.#handleResponseFunctionCallArgumentsDone(event);\n            break;\n          case 'rate_limits.updated':\n            this.#handleRateLimitsUpdated(event);\n            break;\n        }\n      };\n\n      const sendTask = async () => {\n        while (this.#ws && !this.#closing && this.#ws.readyState === WebSocket.OPEN) {\n          try {\n            const event = await this.#sendQueue.get();\n            if (event.type !== 'input_audio_buffer.append') {\n              this.#logger.debug(`-> ${JSON.stringify(this.#loggableEvent(event))}`);\n            }\n            this.#ws.send(JSON.stringify(event));\n          } catch (error) {\n            this.#logger.error('Error sending event:', error);\n          }\n        }\n      };\n\n      sendTask();\n\n      this.#ws.onclose = () => {\n        if (this.#expiresAt && Date.now() >= this.#expiresAt * 1000) {\n          this.#closing = true;\n        }\n        if (!this.#closing) {\n          reject(new Error('OpenAI Realtime connection closed unexpectedly'));\n        }\n        this.#ws = null;\n        resolve();\n      };\n    });\n  }\n\n  async close() {\n    if (!this.#ws) return;\n    this.#closing = true;\n    this.#ws.close();\n    await this.#task;\n  }\n\n  #getContent(ptr: ContentPtr): RealtimeContent {\n    const response = this.#pendingResponses[ptr.response_id];\n    const output = response!.output[ptr.output_index];\n    const content = output!.content[ptr.content_index]!;\n    return content;\n  }\n\n  #handleError(event: api_proto.ErrorEvent): void {\n    this.#logger.error(`OpenAI Realtime error ${JSON.stringify(event.error)}`);\n  }\n\n  #handleSessionCreated(event: api_proto.SessionCreatedEvent): void {\n    this.#sessionId = event.session.id;\n    this.#expiresAt = event.session.expires_at;\n    this.#logger = this.#logger.child({ sessionId: this.#sessionId });\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleSessionUpdated(event: api_proto.SessionUpdatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationCreated(event: api_proto.ConversationCreatedEvent): void {}\n\n  #handleInputAudioBufferCommitted(event: api_proto.InputAudioBufferCommittedEvent): void {\n    this.emit('input_speech_committed', {\n      itemId: event.item_id,\n    } as InputSpeechCommitted);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleInputAudioBufferCleared(event: api_proto.InputAudioBufferClearedEvent): void {}\n\n  #handleInputAudioBufferSpeechStarted(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStartedEvent,\n  ): void {\n    this.emit('input_speech_started', {\n      itemId: event.item_id,\n    } as InputSpeechStarted);\n  }\n\n  #handleInputAudioBufferSpeechStopped(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStoppedEvent,\n  ): void {\n    this.emit('input_speech_stopped');\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {}\n\n  #handleConversationItemInputAudioTranscriptionCompleted(\n    event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n  ): void {\n    const transcript = event.transcript;\n    this.emit('input_speech_transcription_completed', {\n      itemId: event.item_id,\n      transcript: transcript,\n    } as InputSpeechTranscriptionCompleted);\n  }\n\n  #handleConversationItemInputAudioTranscriptionFailed(\n    event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n  ): void {\n    const error = event.error;\n    this.#logger.error(`OpenAI Realtime failed to transcribe input audio: ${error.message}`);\n    this.emit('input_speech_transcription_failed', {\n      itemId: event.item_id,\n      message: error.message,\n    } as InputSpeechTranscriptionFailed);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemTruncated(event: api_proto.ConversationItemTruncatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {}\n\n  #handleResponseCreated(responseCreated: api_proto.ResponseCreatedEvent): void {\n    const response = responseCreated.response;\n    const doneFut = new Future();\n    const newResponse: RealtimeResponse = {\n      id: response.id,\n      status: response.status,\n      statusDetails: response.status_details,\n      usage: null,\n      output: [],\n      doneFut: doneFut,\n      createdTimestamp: Date.now(),\n    };\n    this.#pendingResponses[newResponse.id] = newResponse;\n    this.emit('response_created', newResponse);\n  }\n\n  #handleResponseDone(event: api_proto.ResponseDoneEvent): void {\n    const responseData = event.response;\n    const responseId = responseData.id;\n    const response = this.#pendingResponses[responseId]!;\n    response.status = responseData.status;\n    response.statusDetails = responseData.status_details;\n    response.usage = responseData.usage ?? null;\n    this.#pendingResponses[responseId] = response;\n    response.doneFut.resolve();\n\n    let metricsError: Error | undefined;\n    let cancelled = false;\n    switch (response.status) {\n      case 'failed': {\n        if (response.statusDetails.type !== 'failed') break;\n        const err = response.statusDetails.error;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          code: err?.code,\n          message: err?.message,\n        });\n        this.#logger\n          .child({ code: err?.code, error: err?.message })\n          .error('response generation failed');\n        break;\n      }\n      case 'incomplete': {\n        if (response.statusDetails.type !== 'incomplete') break;\n        const reason = response.statusDetails.reason;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          reason,\n        });\n        this.#logger.child({ reason }).error('response generation incomplete');\n        break;\n      }\n      case 'cancelled': {\n        cancelled = true;\n        break;\n      }\n    }\n    this.emit('response_done', response);\n\n    let ttft: number | undefined;\n    if (response.firstTokenTimestamp) {\n      ttft = response.firstTokenTimestamp - response.createdTimestamp;\n    }\n    const duration = Date.now() - response.createdTimestamp;\n\n    const usage = response.usage;\n    const metric: metrics.MultimodalLLMMetrics = {\n      timestamp: response.createdTimestamp,\n      requestId: response.id,\n      ttft: ttft!,\n      duration,\n      cancelled,\n      label: this.constructor.name,\n      completionTokens: usage?.output_tokens || 0,\n      promptTokens: usage?.input_tokens || 0,\n      totalTokens: usage?.total_tokens || 0,\n      tokensPerSecond: ((usage?.output_tokens || 0) / duration) * 1000,\n      error: metricsError,\n      inputTokenDetails: {\n        cachedTokens: usage?.input_token_details.cached_tokens || 0,\n        textTokens: usage?.input_token_details.text_tokens || 0,\n        audioTokens: usage?.input_token_details.audio_tokens || 0,\n      },\n      outputTokenDetails: {\n        textTokens: usage?.output_token_details.text_tokens || 0,\n        audioTokens: usage?.output_token_details.audio_tokens || 0,\n      },\n    };\n    this.emit('metrics_collected', metric);\n  }\n\n  #handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const itemData = event.item;\n\n    if (itemData.type !== 'message' && itemData.type !== 'function_call') {\n      throw new Error(`Unexpected item type: ${itemData.type}`);\n    }\n\n    let role: api_proto.Role;\n    if (itemData.type === 'function_call') {\n      role = 'assistant'; // function_call doesn't have a role field, defaulting it to assistant\n    } else {\n      role = itemData.role;\n    }\n\n    const newOutput: RealtimeOutput = {\n      responseId: responseId,\n      itemId: itemData.id,\n      outputIndex: event.output_index,\n      type: itemData.type,\n      role: role,\n      content: [],\n      doneFut: new Future(),\n    };\n    response?.output.push(newOutput);\n    this.emit('response_output_added', newOutput);\n  }\n\n  #handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    if (output?.type === 'function_call') {\n      if (!this.#fncCtx) {\n        this.#logger.error('function call received but no fncCtx is available');\n        return;\n      }\n\n      // parse the arguments and call the function inside the fnc_ctx\n      const item = event.item;\n      if (item.type !== 'function_call') {\n        throw new Error('Expected function_call item');\n      }\n      const func = this.#fncCtx[item.name];\n      if (!func) {\n        this.#logger.error(`no function with name ${item.name} in fncCtx`);\n        return;\n      }\n\n      this.emit('function_call_started', {\n        callId: item.call_id,\n      });\n\n      const parsedArgs = JSON.parse(item.arguments);\n\n      this.#logger.debug(\n        `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`,\n      );\n\n      func.execute(parsedArgs).then(\n        (content) => {\n          this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);\n          this.emit('function_call_completed', {\n            callId: item.call_id,\n          });\n          this.conversation.item.create(\n            llm.ChatMessage.createToolFromFunctionResult({\n              name: item.name,\n              toolCallId: item.call_id,\n              result: content,\n            }),\n            output.itemId,\n          );\n          this.response.create();\n        },\n        (error) => {\n          this.#logger.error(`[Function Call ${item.call_id}] ${item.name} failed with ${error}`);\n          // TODO: send it back up as failed?\n          this.emit('function_call_failed', {\n            callId: item.call_id,\n          });\n        },\n      );\n    }\n\n    output?.doneFut.resolve();\n    this.emit('response_output_done', output);\n  }\n\n  #handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    const textStream = new AsyncIterableQueue<string>();\n    const audioStream = new AsyncIterableQueue<AudioFrame>();\n\n    const newContent: RealtimeContent = {\n      responseId: responseId,\n      itemId: event.item_id,\n      outputIndex: outputIndex,\n      contentIndex: event.content_index,\n      text: '',\n      audio: [],\n      textStream: textStream,\n      audioStream: audioStream,\n      toolCalls: [],\n      contentType: event.part.type,\n    };\n    output?.content.push(newContent);\n    response!.firstTokenTimestamp = Date.now();\n    this.emit('response_content_added', newContent);\n  }\n\n  #handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n    const content = this.#getContent(event);\n    this.emit('response_content_done', content);\n  }\n\n  #handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n    this.emit('response_text_delta', event);\n  }\n\n  #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {\n    const content = this.#getContent(event);\n    content.text = event.text;\n    this.emit('response_text_done', event);\n  }\n\n  #handleResponseAudioTranscriptDelta(event: api_proto.ResponseAudioTranscriptDeltaEvent): void {\n    const content = this.#getContent(event);\n    const transcript = event.delta;\n    content.text += transcript;\n\n    content.textStream.put(transcript);\n  }\n\n  #handleResponseAudioTranscriptDone(event: api_proto.ResponseAudioTranscriptDoneEvent): void {\n    const content = this.#getContent(event);\n    content.textStream.close();\n  }\n\n  #handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n    const content = this.#getContent(event);\n    const data = Buffer.from(event.delta, 'base64');\n    const audio = new AudioFrame(\n      new Int16Array(data.buffer),\n      api_proto.SAMPLE_RATE,\n      api_proto.NUM_CHANNELS,\n      data.length / 2,\n    );\n    content.audio.push(audio);\n\n    content.audioStream.put(audio);\n  }\n\n  #handleResponseAudioDone(event: api_proto.ResponseAudioDoneEvent): void {\n    const content = this.#getContent(event);\n    content.audioStream.close();\n  }\n\n  #handleResponseFunctionCallArgumentsDelta(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDeltaEvent,\n  ): void {}\n\n  #handleResponseFunctionCallArgumentsDone(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDoneEvent,\n  ): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleRateLimitsUpdated(event: api_proto.RateLimitsUpdatedEvent): void {}\n}\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,YAAY,eAAe;AA8E3B,MAAM,iBAAiB;AAAA,EACrB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,OAAO,OAAmB;AACxB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,OAAO,OAAO,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,IACzD,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAEA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS,QAAgB,cAAsB,UAAkB;AAC/D,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,MACT,eAAe;AAAA,MACf,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,QAAgB;AACrB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,SAA0B,gBAA+B;AAC9D,QAAI,CAAC,QAAQ,SAAS;AACpB;AAAA,IACF;AAEA,QAAI;AAEJ,QAAI,QAAQ,YAAY;AACtB,UAAI,OAAO,QAAQ,YAAY,UAAU;AACvC,cAAM,IAAI,UAAU,kCAAkC;AAAA,MACxD;AAEA,cAAQ;AAAA,QACN,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,SAAS,QAAQ;AAAA,UACjB,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,IACF,OAAO;AACL,UAAI,UAAU,QAAQ;AACtB,UAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B,kBAAU,CAAC,OAAO;AAAA,MACpB;AAEA,UAAI,QAAQ,SAAS,IAAI,SAAS,MAAM;AACtC,cAAM,WAAyE,CAAC;AAChF,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,OAAO,OAAO,KAAK,YAAY,EAAE,KAAK,EAAE,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,IAAI,SAAS,WAAW;AAClD,cAAM,WAAoC,CAAC;AAC3C,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,qDAAqD;AAAA,UACzE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,IAAI,SAAS,QAAQ;AAC/C,cAAM,WAAyC,CAAC;AAChD,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,kDAAkD;AAAA,UACtE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,OAAO;AACL,aAAK,QACF,MAAM,EAAE,QAAQ,CAAC,EACjB,KAAK,uDAAuD;AAC/D;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,SAAS,KAAK;AAAA,EAC9B;AACF;AAEA,MAAM,aAAa;AAAA,EACjB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,IAAI,OAAyB;AAC3B,WAAO,IAAI,iBAAiB,KAAK,QAAQ;AAAA,EAC3C;AACF;AAEA,MAAM,SAAS;AAAA,EACb;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAQO,MAAM,sBAAsB,WAAW,cAAc;AAAA,EAC1D,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA,EAEzB;AAAA,EACA,YAA+B,CAAC;AAAA,EAEhC,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA,aAAa;AAAA,IACb,SAAS;AAAA,IACT,aAAa;AAAA,IACb,eAAe;AAAA,IACf,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,EAC5B,GAeG;AACD,WAAO,IAAI,cAAc;AAAA,MACvB,SAAS;AAAA,MACT,SAAS,IAAI,IAAI,UAAU,OAAO,EAAE,SAAS;AAAA,MAC7C,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,YAAY;AAAA,IACV,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,IAC1B,QAAQ;AAAA,IACR,SAAS,QAAQ,IAAI,kBAAkB;AAAA,IACvC,UAAU,UAAU;AAAA;AAAA,IAEpB,UAAU;AAAA,IACV,aAAa;AAAA,IACb,aAAa;AAAA,EACf,GAgBG;AACD,UAAM;AAEN,QAAI,WAAW,IAAI;AACjB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,SAAK,eAAe;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,WAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AAAA,IACN;AAAA,IACA;AAAA,IACA,aAAa,KAAK,aAAa;AAAA,IAC/B,eAAe,KAAK,aAAa;AAAA,IACjC,QAAQ,KAAK,aAAa;AAAA,IAC1B,mBAAmB,KAAK,aAAa;AAAA,IACrC,oBAAoB,KAAK,aAAa;AAAA,IACtC,0BAA0B,KAAK,aAAa;AAAA,IAC5C,gBAAgB,KAAK,aAAa;AAAA,IAClC,cAAc,KAAK,aAAa;AAAA,IAChC,0BAA0B,KAAK,aAAa;AAAA,EAC9C,GAYoB;AAClB,UAAM,OAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,aAAa;AAAA,MACzB,QAAQ,KAAK,aAAa;AAAA,MAC1B,SAAS,KAAK,aAAa;AAAA,MAC3B,SAAS,KAAK,aAAa;AAAA,MAC3B,YAAY,KAAK,aAAa;AAAA,MAC9B,YAAY,KAAK,aAAa;AAAA,IAChC;AAEA,UAAM,aAAa,IAAI,gBAAgB,MAAM;AAAA,MAC3C,SAAS,WAAW,IAAI,IAAI,YAAY;AAAA,MACxC;AAAA,IACF,CAAC;AACD,SAAK,UAAU,KAAK,UAAU;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,QAAQ,WAAW,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC3E;AACF;AAEO,MAAM,wBAAwB,WAAW,gBAAgB;AAAA,EAC9D,WAAwC;AAAA,EACxC,UAA2C;AAAA,EAC3C;AAAA,EACA,oBAAwD,CAAC;AAAA,EACzD,aAAa;AAAA,EACb,MAAwB;AAAA,EACxB,aAA4B;AAAA,EAC5B,UAAU,IAAI;AAAA,EACd;AAAA,EACA,WAAW;AAAA,EACX,aAAa,IAAI,MAA6B;AAAA,EAE9C,YACE,MACA,EAAE,QAAQ,QAAQ,GAClB;AACA,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAEf,SAAK,QAAQ,KAAK,OAAO;AAEzB,SAAK,cAAc;AAAA,MACjB,YAAY,KAAK,MAAM;AAAA,MACvB,cAAc,KAAK,MAAM;AAAA,MACzB,OAAO,KAAK,MAAM;AAAA,MAClB,kBAAkB,KAAK,MAAM;AAAA,MAC7B,mBAAmB,KAAK,MAAM;AAAA,MAC9B,yBAAyB,KAAK,MAAM;AAAA,MACpC,eAAe,KAAK,MAAM;AAAA,MAC1B,aAAa,KAAK,MAAM;AAAA,MACxB,yBAAyB,KAAK,MAAM;AAAA,MACpC,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAAA,EAEA,IAAI,UAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,IAAI,eAA6B;AAC/B,WAAO,IAAI,aAAa,IAAI;AAAA,EAC9B;AAAA,EAEA,IAAI,mBAAqC;AACvC,WAAO,IAAI,iBAAiB,IAAI;AAAA,EAClC;AAAA,EAEA,IAAI,WAAqB;AACvB,WAAO,IAAI,SAAS,IAAI;AAAA,EAC1B;AAAA,EAEA,IAAI,aAAqB;AACvB,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,qBAAqB;AAAA,IACvC;AACA,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,SAAS,SAAsC;AAC7C,SAAK,WAAW,IAAI,OAAO;AAAA,EAC7B;AAAA;AAAA;AAAA,EAIA,eACE,OACA,YAAoB,IACK;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,YAAM,gBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,cAAc;AAAA,IACjD;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,YAAM,iBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,eAAe;AAAA,IAClD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc;AAAA,IACZ,aAAa,KAAK,MAAM;AAAA,IACxB,eAAe,KAAK,MAAM;AAAA,IAC1B,QAAQ,KAAK,MAAM;AAAA,IACnB,mBAAmB,KAAK,MAAM;AAAA,IAC9B,oBAAoB,KAAK,MAAM;AAAA,IAC/B,0BAA0B,KAAK,MAAM;AAAA,IACrC,gBAAgB,KAAK,MAAM;AAAA,IAC3B,cAAc,KAAK,MAAM;AAAA,IACzB,0BAA0B,KAAK,MAAM;AAAA,IACrC,aAAa;AAAA,EACf,GAWG;AACD,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,MAAM;AAAA,MAClB,QAAQ,KAAK,MAAM;AAAA,MACnB,SAAS,KAAK,MAAM;AAAA,MACpB,SAAS,KAAK,MAAM;AAAA,MACpB,YAAY,KAAK,MAAM;AAAA,MACvB,YAAY,KAAK,MAAM;AAAA,IACzB;AAEA,UAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MAClD,MAAM;AAAA,MACN;AAAA,MACA,aAAa,KAAK;AAAA,MAClB;AAAA;AAAA,QAEE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,IAAI,UAAU,KAAK,UAAU;AAAA;AAAA,IACrC,EAAE,IACF,CAAC;AAEL,UAAM,qBAAmD;AAAA,MACvD,MAAM;AAAA,MACN,SAAS;AAAA,QACP,YAAY,KAAK,MAAM;AAAA,QACvB,cAAc,KAAK,MAAM;AAAA,QACzB,OAAO,KAAK,MAAM;AAAA,QAClB,oBAAoB,KAAK,MAAM;AAAA,QAC/B,qBAAqB,KAAK,MAAM;AAAA,QAChC,2BAA2B,KAAK,MAAM;AAAA,QACtC,gBAAgB,KAAK,MAAM;AAAA,QAC3B,aAAa,KAAK,MAAM;AAAA,QACxB,4BACE,KAAK,MAAM,4BAA4B,WACnC,QACA,KAAK,MAAM;AAAA,QACjB;AAAA,QACA,aAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,WAAW,KAAK,MAAM,4BAA4B,UAAU;AAEzE,yBAAmB,QAAQ,6BAA6B;AAAA,IAC1D;AAEA,SAAK,SAAS,kBAAkB;AAAA,EAClC;AAAA;AAAA,EAGA,6BAA6B,UAAmC;AAC9D,UAAM,UAAU,WAAW,UAAU;AACrC,WAAO,IAAI,IAAI,YAAY;AAAA,MACzB,MAAM,IAAI,SAAS;AAAA,MACnB,SAAS;AAAA,QACP,OAAO,IAAI;AAAA,UACT,IAAI,WAAW,UAAU,UAAU,YAAY;AAAA,UAC/C,UAAU;AAAA,UACV,UAAU;AAAA,UACV;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,wBAAwB,QAAgB;AACtC,QAAI,QAAQ;AACV,WAAK,aAAa,KAAK,OAAO,MAAM;AAAA,IACtC;AACA,SAAK,aAAa,KAAK,OAAO,KAAK,6BAA6B,CAAC,CAAC;AAClE,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,SAAwB;AACtB,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAC5C,YAAM,UAAkC;AAAA,QACtC,cAAc;AAAA,MAChB;AACA,UAAI,KAAK,MAAM,SAAS;AAItB,YAAI,KAAK,MAAM,YAAY;AACzB,kBAAQ,gBAAgB,UAAU,KAAK,MAAM,UAAU;AAAA,QACzD,WAAW,KAAK,MAAM,QAAQ;AAC5B,kBAAQ,SAAS,IAAI,KAAK,MAAM;AAAA,QAClC,OAAO;AACL,iBAAO,IAAI,MAAM,6CAA6C,CAAC;AAC/D;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,gBAAgB,UAAU,KAAK,MAAM,MAAM;AACnD,gBAAQ,aAAa,IAAI;AAAA,MAC3B;AACA,YAAM,MAAM,IAAI,IAAI,CAAC,KAAK,MAAM,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAC9D,UAAI,IAAI,aAAa,UAAU;AAC7B,YAAI,WAAW;AAAA,MACjB;AAGA,YAAM,cAAsC,CAAC;AAC7C,UAAI,KAAK,MAAM,SAAS;AACtB,oBAAY,aAAa,IAAI;AAC7B,oBAAY,YAAY,IAAI,KAAK,MAAM;AAAA,MACzC,OAAO;AACL,oBAAY,OAAO,IAAI,KAAK,MAAM;AAAA,MACpC;AAEA,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,YAAI,aAAa,IAAI,KAAK,KAAK;AAAA,MACjC;AAEA,cAAQ,MAAM,yCAAyC,IAAI,SAAS,CAAC;AACrE,WAAK,MAAM,IAAI,UAAU,IAAI,SAAS,GAAG;AAAA,QACvC;AAAA,MACF,CAAC;AAED,WAAK,IAAI,UAAU,CAAC,UAAU;AAC5B,eAAO,IAAI,MAAM,sCAAsC,MAAM,OAAO,CAAC;AAAA,MACvE;AAEA,YAAM,KAAK,KAAK,KAAK,MAAM;AAC3B,WAAK,WAAW;AAEhB,WAAK,IAAI,YAAY,CAAC,YAAY;AAChC,cAAM,QAA+B,KAAK,MAAM,QAAQ,IAAc;AACtE,aAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AACrE,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,aAAa,KAAK;AACvB;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,2BAA2B,KAAK;AACrC;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,wDAAwD,KAAK;AAClE;AAAA,UACF,KAAK;AACH,iBAAK,qDAAqD,KAAK;AAC/D;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,uBAAuB,KAAK;AACjC;AAAA,UACF,KAAK;AACH,iBAAK,oBAAoB,KAAK;AAC9B;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,8BAA8B,KAAK;AACxC;AAAA,UACF,KAAK;AACH,iBAAK,gCAAgC,KAAK;AAC1C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,wBAAwB,KAAK;AAClC;AAAA,UACF,KAAK;AACH,iBAAK,oCAAoC,KAAK;AAC9C;AAAA,UACF,KAAK;AACH,iBAAK,mCAAmC,KAAK;AAC7C;AAAA,UACF,KAAK;AACH,iBAAK,0BAA0B,KAAK;AACpC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,0CAA0C,KAAK;AACpD;AAAA,UACF,KAAK;AACH,iBAAK,yCAAyC,KAAK;AACnD;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,QACJ;AAAA,MACF;AAEA,YAAM,WAAW,YAAY;AAC3B,eAAO,KAAK,OAAO,CAAC,KAAK,YAAY,KAAK,IAAI,eAAe,UAAU,MAAM;AAC3E,cAAI;AACF,kBAAM,QAAQ,MAAM,KAAK,WAAW,IAAI;AACxC,gBAAI,MAAM,SAAS,6BAA6B;AAC9C,mBAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AAAA,YACvE;AACA,iBAAK,IAAI,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,UACrC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,wBAAwB,KAAK;AAAA,UAClD;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAET,WAAK,IAAI,UAAU,MAAM;AACvB,YAAI,KAAK,cAAc,KAAK,IAAI,KAAK,KAAK,aAAa,KAAM;AAC3D,eAAK,WAAW;AAAA,QAClB;AACA,YAAI,CAAC,KAAK,UAAU;AAClB,iBAAO,IAAI,MAAM,gDAAgD,CAAC;AAAA,QACpE;AACA,aAAK,MAAM;AACX,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,CAAC,KAAK,IAAK;AACf,SAAK,WAAW;AAChB,SAAK,IAAI,MAAM;AACf,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,YAAY,KAAkC;AAC5C,UAAM,WAAW,KAAK,kBAAkB,IAAI,WAAW;AACvD,UAAM,SAAS,SAAU,OAAO,IAAI,YAAY;AAChD,UAAM,UAAU,OAAQ,QAAQ,IAAI,aAAa;AACjD,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmC;AAC9C,SAAK,QAAQ,MAAM,yBAAyB,KAAK,UAAU,MAAM,KAAK,CAAC,EAAE;AAAA,EAC3E;AAAA,EAEA,sBAAsB,OAA4C;AAChE,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,UAAU,KAAK,QAAQ,MAAM,EAAE,WAAW,KAAK,WAAW,CAAC;AAAA,EAClE;AAAA;AAAA,EAGA,sBAAsB,OAA4C;AAAA,EAAC;AAAA;AAAA,EAGnE,2BAA2B,OAAiD;AAAA,EAAC;AAAA,EAE7E,iCAAiC,OAAuD;AACtF,SAAK,KAAK,0BAA0B;AAAA,MAClC,QAAQ,MAAM;AAAA,IAChB,CAAyB;AAAA,EAC3B;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,qCAEE,OACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,QAAQ,MAAM;AAAA,IAChB,CAAuB;AAAA,EACzB;AAAA,EAEA,qCAEE,OACM;AACN,SAAK,KAAK,sBAAsB;AAAA,EAClC;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,wDACE,OACM;AACN,UAAM,aAAa,MAAM;AACzB,SAAK,KAAK,wCAAwC;AAAA,MAChD,QAAQ,MAAM;AAAA,MACd;AAAA,IACF,CAAsC;AAAA,EACxC;AAAA,EAEA,qDACE,OACM;AACN,UAAM,QAAQ,MAAM;AACpB,SAAK,QAAQ,MAAM,qDAAqD,MAAM,OAAO,EAAE;AACvF,SAAK,KAAK,qCAAqC;AAAA,MAC7C,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM;AAAA,IACjB,CAAmC;AAAA,EACrC;AAAA;AAAA,EAGA,iCAAiC,OAAuD;AAAA,EAAC;AAAA;AAAA,EAGzF,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,uBAAuB,iBAAuD;AAC5E,UAAM,WAAW,gBAAgB;AACjC,UAAM,UAAU,IAAI,OAAO;AAC3B,UAAM,cAAgC;AAAA,MACpC,IAAI,SAAS;AAAA,MACb,QAAQ,SAAS;AAAA,MACjB,eAAe,SAAS;AAAA,MACxB,OAAO;AAAA,MACP,QAAQ,CAAC;AAAA,MACT;AAAA,MACA,kBAAkB,KAAK,IAAI;AAAA,IAC7B;AACA,SAAK,kBAAkB,YAAY,EAAE,IAAI;AACzC,SAAK,KAAK,oBAAoB,WAAW;AAAA,EAC3C;AAAA,EAEA,oBAAoB,OAA0C;AAC5D,UAAM,eAAe,MAAM;AAC3B,UAAM,aAAa,aAAa;AAChC,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,aAAS,SAAS,aAAa;AAC/B,aAAS,gBAAgB,aAAa;AACtC,aAAS,QAAQ,aAAa,SAAS;AACvC,SAAK,kBAAkB,UAAU,IAAI;AACrC,aAAS,QAAQ,QAAQ;AAEzB,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,SAAS,QAAQ;AAAA,MACvB,KAAK,UAAU;AACb,YAAI,SAAS,cAAc,SAAS,SAAU;AAC9C,cAAM,MAAM,SAAS,cAAc;AACnC,uBAAe,IAAI,QAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B,MAAM,2BAAK;AAAA,UACX,SAAS,2BAAK;AAAA,QAChB,CAAC;AACD,aAAK,QACF,MAAM,EAAE,MAAM,2BAAK,MAAM,OAAO,2BAAK,QAAQ,CAAC,EAC9C,MAAM,4BAA4B;AACrC;AAAA,MACF;AAAA,MACA,KAAK,cAAc;AACjB,YAAI,SAAS,cAAc,SAAS,aAAc;AAClD,cAAM,SAAS,SAAS,cAAc;AACtC,uBAAe,IAAI,QAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B;AAAA,QACF,CAAC;AACD,aAAK,QAAQ,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,gCAAgC;AACrE;AAAA,MACF;AAAA,MACA,KAAK,aAAa;AAChB,oBAAY;AACZ;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAK,iBAAiB,QAAQ;AAEnC,QAAI;AACJ,QAAI,SAAS,qBAAqB;AAChC,aAAO,SAAS,sBAAsB,SAAS;AAAA,IACjD;AACA,UAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AAEvC,UAAM,QAAQ,SAAS;AACvB,UAAM,SAAuC;AAAA,MAC3C,WAAW,SAAS;AAAA,MACpB,WAAW,SAAS;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,YAAY;AAAA,MACxB,mBAAkB,+BAAO,kBAAiB;AAAA,MAC1C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,mBAAmB,+BAAO,kBAAiB,KAAK,WAAY;AAAA,MAC5D,OAAO;AAAA,MACP,mBAAmB;AAAA,QACjB,eAAc,+BAAO,oBAAoB,kBAAiB;AAAA,QAC1D,aAAY,+BAAO,oBAAoB,gBAAe;AAAA,QACtD,cAAa,+BAAO,oBAAoB,iBAAgB;AAAA,MAC1D;AAAA,MACA,oBAAoB;AAAA,QAClB,aAAY,+BAAO,qBAAqB,gBAAe;AAAA,QACvD,cAAa,+BAAO,qBAAqB,iBAAgB;AAAA,MAC3D;AAAA,IACF;AACA,SAAK,KAAK,qBAAqB,MAAM;AAAA,EACvC;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,WAAW,MAAM;AAEvB,QAAI,SAAS,SAAS,aAAa,SAAS,SAAS,iBAAiB;AACpE,YAAM,IAAI,MAAM,yBAAyB,SAAS,IAAI,EAAE;AAAA,IAC1D;AAEA,QAAI;AACJ,QAAI,SAAS,SAAS,iBAAiB;AACrC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,SAAS;AAAA,IAClB;AAEA,UAAM,YAA4B;AAAA,MAChC;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,aAAa,MAAM;AAAA,MACnB,MAAM,SAAS;AAAA,MACf;AAAA,MACA,SAAS,CAAC;AAAA,MACV,SAAS,IAAI,OAAO;AAAA,IACtB;AACA,yCAAU,OAAO,KAAK;AACtB,SAAK,KAAK,yBAAyB,SAAS;AAAA,EAC9C;AAAA,EAEA,8BAA8B,OAAoD;AAChF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,SAAI,iCAAQ,UAAS,iBAAiB;AACpC,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,QAAQ,MAAM,mDAAmD;AACtE;AAAA,MACF;AAGA,YAAM,OAAO,MAAM;AACnB,UAAI,KAAK,SAAS,iBAAiB;AACjC,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,YAAM,OAAO,KAAK,QAAQ,KAAK,IAAI;AACnC,UAAI,CAAC,MAAM;AACT,aAAK,QAAQ,MAAM,yBAAyB,KAAK,IAAI,YAAY;AACjE;AAAA,MACF;AAEA,WAAK,KAAK,yBAAyB;AAAA,QACjC,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,aAAa,KAAK,MAAM,KAAK,SAAS;AAE5C,WAAK,QAAQ;AAAA,QACX,kBAAkB,KAAK,OAAO,eAAe,KAAK,IAAI,mBAAmB,UAAU;AAAA,MACrF;AAEA,WAAK,QAAQ,UAAU,EAAE;AAAA,QACvB,CAAC,YAAY;AACX,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,aAAa,OAAO,EAAE;AACrF,eAAK,KAAK,2BAA2B;AAAA,YACnC,QAAQ,KAAK;AAAA,UACf,CAAC;AACD,eAAK,aAAa,KAAK;AAAA,YACrB,IAAI,YAAY,6BAA6B;AAAA,cAC3C,MAAM,KAAK;AAAA,cACX,YAAY,KAAK;AAAA,cACjB,QAAQ;AAAA,YACV,CAAC;AAAA,YACD,OAAO;AAAA,UACT;AACA,eAAK,SAAS,OAAO;AAAA,QACvB;AAAA,QACA,CAAC,UAAU;AACT,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,gBAAgB,KAAK,EAAE;AAEtF,eAAK,KAAK,wBAAwB;AAAA,YAChC,QAAQ,KAAK;AAAA,UACf,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,qCAAQ,QAAQ;AAChB,SAAK,KAAK,wBAAwB,MAAM;AAAA,EAC1C;AAAA,EAEA,gCAAgC,OAAsD;AACpF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,UAAM,aAAa,IAAI,mBAA2B;AAClD,UAAM,cAAc,IAAI,mBAA+B;AAEvD,UAAM,aAA8B;AAAA,MAClC;AAAA,MACA,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,cAAc,MAAM;AAAA,MACpB,MAAM;AAAA,MACN,OAAO,CAAC;AAAA,MACR;AAAA,MACA;AAAA,MACA,WAAW,CAAC;AAAA,MACZ,aAAa,MAAM,KAAK;AAAA,IAC1B;AACA,qCAAQ,QAAQ,KAAK;AACrB,aAAU,sBAAsB,KAAK,IAAI;AACzC,SAAK,KAAK,0BAA0B,UAAU;AAAA,EAChD;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,SAAK,KAAK,yBAAyB,OAAO;AAAA,EAC5C;AAAA,EAEA,yBAAyB,OAA+C;AACtE,SAAK,KAAK,uBAAuB,KAAK;AAAA,EACxC;AAAA,EAEA,wBAAwB,OAA8C;AACpE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,OAAO,MAAM;AACrB,SAAK,KAAK,sBAAsB,KAAK;AAAA,EACvC;AAAA,EAEA,oCAAoC,OAA0D;AAC5F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,aAAa,MAAM;AACzB,YAAQ,QAAQ;AAEhB,YAAQ,WAAW,IAAI,UAAU;AAAA,EACnC;AAAA,EAEA,mCAAmC,OAAyD;AAC1F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,WAAW,MAAM;AAAA,EAC3B;AAAA,EAEA,0BAA0B,OAAgD;AACxE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO,QAAQ;AAC9C,UAAM,QAAQ,IAAI;AAAA,MAChB,IAAI,WAAW,KAAK,MAAM;AAAA,MAC1B,UAAU;AAAA,MACV,UAAU;AAAA,MACV,KAAK,SAAS;AAAA,IAChB;AACA,YAAQ,MAAM,KAAK,KAAK;AAExB,YAAQ,YAAY,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,yBAAyB,OAA+C;AACtE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,YAAY,MAAM;AAAA,EAC5B;AAAA,EAEA,0CAEE,OACM;AAAA,EAAC;AAAA,EAET,yCAEE,OACM;AAAA,EAAC;AAAA;AAAA,EAGT,yBAAyB,OAA+C;AAAA,EAAC;AAC3E;","names":["c"]}
+{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n  AsyncIterableQueue,\n  Future,\n  Queue,\n  llm,\n  log,\n  mergeFrames,\n  metrics,\n  multimodal,\n} from '@livekit/agents';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { once } from 'node:events';\nimport { WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\ninterface ModelOptions {\n  modalities: ['text', 'audio'] | ['text'];\n  instructions: string;\n  voice: api_proto.Voice;\n  inputAudioFormat: api_proto.AudioFormat;\n  outputAudioFormat: api_proto.AudioFormat;\n  inputAudioTranscription: api_proto.InputAudioTranscription | null;\n  turnDetection: api_proto.TurnDetectionType | null;\n  temperature: number;\n  maxResponseOutputTokens: number;\n  model: api_proto.Model;\n  apiKey?: string;\n  baseURL: string;\n  isAzure: boolean;\n  entraToken?: string;\n  apiVersion?: string;\n}\n\nexport interface RealtimeResponse {\n  id: string;\n  status: api_proto.ResponseStatus;\n  statusDetails: api_proto.ResponseStatusDetails | null;\n  usage: api_proto.ModelUsage | null;\n  output: RealtimeOutput[];\n  doneFut: Future;\n  createdTimestamp: number;\n  firstTokenTimestamp?: number;\n}\n\nexport interface RealtimeOutput {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  role: api_proto.Role;\n  type: 'message' | 'function_call';\n  content: RealtimeContent[];\n  doneFut: Future;\n}\n\nexport interface RealtimeContent {\n  responseId: string;\n  itemId: string;\n  outputIndex: number;\n  contentIndex: number;\n  text: string;\n  audio: AudioFrame[];\n  textStream: AsyncIterableQueue<string>;\n  audioStream: AsyncIterableQueue<AudioFrame>;\n  toolCalls: RealtimeToolCall[];\n  contentType: api_proto.Modality;\n}\n\nexport interface RealtimeToolCall {\n  name: string;\n  arguments: string;\n  toolCallID: string;\n}\n\nexport interface InputSpeechTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n}\n\nexport interface InputSpeechTranscriptionFailed {\n  itemId: string;\n  message: string;\n}\n\nexport interface InputSpeechStarted {\n  itemId: string;\n}\n\nexport interface InputSpeechCommitted {\n  itemId: string;\n}\n\nclass InputAudioBuffer {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  append(frame: AudioFrame) {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.append',\n      audio: Buffer.from(frame.data.buffer).toString('base64'),\n    });\n  }\n\n  clear() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.clear',\n    });\n  }\n\n  commit() {\n    this.#session.queueMsg({\n      type: 'input_audio_buffer.commit',\n    });\n  }\n}\n\nclass ConversationItem {\n  #session: RealtimeSession;\n  #logger = log();\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  truncate(itemId: string, contentIndex: number, audioEnd: number) {\n    this.#session.queueMsg({\n      type: 'conversation.item.truncate',\n      item_id: itemId,\n      content_index: contentIndex,\n      audio_end_ms: audioEnd,\n    });\n  }\n\n  delete(itemId: string) {\n    this.#session.queueMsg({\n      type: 'conversation.item.delete',\n      item_id: itemId,\n    });\n  }\n\n  create(message: llm.ChatMessage, previousItemId?: string): void {\n    if (!message.content) {\n      return;\n    }\n\n    let event: api_proto.ConversationItemCreateEvent;\n\n    if (message.toolCallId) {\n      if (typeof message.content !== 'string') {\n        throw new TypeError('message.content must be a string');\n      }\n\n      event = {\n        type: 'conversation.item.create',\n        previous_item_id: previousItemId,\n        item: {\n          type: 'function_call_output',\n          call_id: message.toolCallId,\n          output: message.content,\n        },\n      };\n    } else {\n      let content = message.content;\n      if (!Array.isArray(content)) {\n        content = [content];\n      }\n\n      if (message.role === llm.ChatRole.USER) {\n        const contents: (api_proto.InputTextContent | api_proto.InputAudioContent)[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            contents.push({\n              type: 'input_audio',\n              audio: Buffer.from(mergeFrames(c.frame).data.buffer).toString('base64'),\n            });\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'user',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.ASSISTANT) {\n        const contents: api_proto.TextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in assistant message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'assistant',\n            content: contents,\n          },\n        };\n      } else if (message.role === llm.ChatRole.SYSTEM) {\n        const contents: api_proto.InputTextContent[] = [];\n        for (const c of content) {\n          if (typeof c === 'string') {\n            contents.push({\n              type: 'input_text',\n              text: c,\n            });\n          } else if (\n            // typescript type guard for determining ChatAudio vs ChatImage\n            ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatAudio => {\n              return (c as llm.ChatAudio).frame !== undefined;\n            })(c)\n          ) {\n            this.#logger.warn('audio content in system message is not supported');\n          }\n        }\n\n        event = {\n          type: 'conversation.item.create',\n          previous_item_id: previousItemId,\n          item: {\n            type: 'message',\n            role: 'system',\n            content: contents,\n          },\n        };\n      } else {\n        this.#logger\n          .child({ message })\n          .warn('chat message is not supported inside the realtime API');\n        return;\n      }\n    }\n\n    this.#session.queueMsg(event);\n  }\n}\n\nclass Conversation {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  get item(): ConversationItem {\n    return new ConversationItem(this.#session);\n  }\n}\n\nclass Response {\n  #session: RealtimeSession;\n\n  constructor(session: RealtimeSession) {\n    this.#session = session;\n  }\n\n  create() {\n    this.#session.queueMsg({\n      type: 'response.create',\n    });\n  }\n\n  cancel() {\n    this.#session.queueMsg({\n      type: 'response.cancel',\n    });\n  }\n}\n\ninterface ContentPtr {\n  response_id: string;\n  output_index: number;\n  content_index: number;\n}\n\nexport class RealtimeModel extends multimodal.RealtimeModel {\n  sampleRate = api_proto.SAMPLE_RATE;\n  numChannels = api_proto.NUM_CHANNELS;\n  inFrameSize = api_proto.IN_FRAME_SIZE;\n  outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n  #defaultOpts: ModelOptions;\n  #sessions: RealtimeSession[] = [];\n\n  static withAzure({\n    baseURL,\n    azureDeployment,\n    apiVersion = '2024-10-01-preview',\n    apiKey = undefined,\n    entraToken = undefined,\n    instructions = '',\n    modalities = ['text', 'audio'],\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n  }: {\n    baseURL: string;\n    azureDeployment: string;\n    apiVersion?: string;\n    apiKey?: string;\n    entraToken?: string;\n    instructions?: string;\n    modalities?: ['text', 'audio'] | ['text'];\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }) {\n    return new RealtimeModel({\n      isAzure: true,\n      baseURL: new URL('openai', baseURL).toString(),\n      model: azureDeployment,\n      apiVersion,\n      apiKey,\n      entraToken,\n      instructions,\n      modalities,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n    });\n  }\n\n  constructor({\n    modalities = ['text', 'audio'],\n    instructions = '',\n    voice = 'alloy',\n    inputAudioFormat = 'pcm16',\n    outputAudioFormat = 'pcm16',\n    inputAudioTranscription = { model: 'whisper-1' },\n    turnDetection = { type: 'server_vad' },\n    temperature = 0.8,\n    maxResponseOutputTokens = Infinity,\n    model = 'gpt-4o-realtime-preview-2024-10-01',\n    apiKey = process.env.OPENAI_API_KEY || '',\n    baseURL = api_proto.BASE_URL,\n    // used for microsoft\n    isAzure = false,\n    apiVersion = undefined,\n    entraToken = undefined,\n  }: {\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription;\n    turnDetection?: api_proto.TurnDetectionType;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    model?: api_proto.Model;\n    apiKey?: string;\n    baseURL?: string;\n    isAzure?: boolean;\n    apiVersion?: string;\n    entraToken?: string;\n  }) {\n    super();\n\n    if (apiKey === '') {\n      throw new Error(\n        'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable',\n      );\n    }\n\n    this.#defaultOpts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model,\n      apiKey,\n      baseURL,\n      isAzure,\n      apiVersion,\n      entraToken,\n    };\n  }\n\n  get sessions(): RealtimeSession[] {\n    return this.#sessions;\n  }\n\n  session({\n    fncCtx,\n    chatCtx,\n    modalities = this.#defaultOpts.modalities,\n    instructions = this.#defaultOpts.instructions,\n    voice = this.#defaultOpts.voice,\n    inputAudioFormat = this.#defaultOpts.inputAudioFormat,\n    outputAudioFormat = this.#defaultOpts.outputAudioFormat,\n    inputAudioTranscription = this.#defaultOpts.inputAudioTranscription,\n    turnDetection = this.#defaultOpts.turnDetection,\n    temperature = this.#defaultOpts.temperature,\n    maxResponseOutputTokens = this.#defaultOpts.maxResponseOutputTokens,\n  }: {\n    fncCtx?: llm.FunctionContext;\n    chatCtx?: llm.ChatContext;\n    modalities?: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n  }): RealtimeSession {\n    const opts: ModelOptions = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#defaultOpts.model,\n      apiKey: this.#defaultOpts.apiKey,\n      baseURL: this.#defaultOpts.baseURL,\n      isAzure: this.#defaultOpts.isAzure,\n      apiVersion: this.#defaultOpts.apiVersion,\n      entraToken: this.#defaultOpts.entraToken,\n    };\n\n    const newSession = new RealtimeSession(opts, {\n      chatCtx: chatCtx || new llm.ChatContext(),\n      fncCtx,\n    });\n    this.#sessions.push(newSession);\n    return newSession;\n  }\n\n  async close() {\n    await Promise.allSettled(this.#sessions.map((session) => session.close()));\n  }\n}\n\nexport class RealtimeSession extends multimodal.RealtimeSession {\n  #chatCtx: llm.ChatContext | undefined = undefined;\n  #fncCtx: llm.FunctionContext | undefined = undefined;\n  #opts: ModelOptions;\n  #pendingResponses: { [id: string]: RealtimeResponse } = {};\n  #sessionId = 'not-connected';\n  #ws: WebSocket | null = null;\n  #expiresAt: number | null = null;\n  #logger = log();\n  #task: Promise<void>;\n  #closing = true;\n  #sendQueue = new Queue<api_proto.ClientEvent>();\n\n  constructor(\n    opts: ModelOptions,\n    { fncCtx, chatCtx }: { fncCtx?: llm.FunctionContext; chatCtx?: llm.ChatContext },\n  ) {\n    super();\n\n    this.#opts = opts;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n\n    this.#task = this.#start();\n\n    this.sessionUpdate({\n      modalities: this.#opts.modalities,\n      instructions: this.#opts.instructions,\n      voice: this.#opts.voice,\n      inputAudioFormat: this.#opts.inputAudioFormat,\n      outputAudioFormat: this.#opts.outputAudioFormat,\n      inputAudioTranscription: this.#opts.inputAudioTranscription,\n      turnDetection: this.#opts.turnDetection,\n      temperature: this.#opts.temperature,\n      maxResponseOutputTokens: this.#opts.maxResponseOutputTokens,\n      toolChoice: 'auto',\n    });\n  }\n\n  get chatCtx(): llm.ChatContext | undefined {\n    return this.#chatCtx;\n  }\n\n  get fncCtx(): llm.FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  set fncCtx(ctx: llm.FunctionContext | undefined) {\n    this.#fncCtx = ctx;\n  }\n\n  get conversation(): Conversation {\n    return new Conversation(this);\n  }\n\n  get inputAudioBuffer(): InputAudioBuffer {\n    return new InputAudioBuffer(this);\n  }\n\n  get response(): Response {\n    return new Response(this);\n  }\n\n  get expiration(): number {\n    if (!this.#expiresAt) {\n      throw new Error('session not started');\n    }\n    return this.#expiresAt * 1000;\n  }\n\n  queueMsg(command: api_proto.ClientEvent): void {\n    this.#sendQueue.put(command);\n  }\n\n  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs\n  /// with large amounts of base64 audio data.\n  #loggableEvent(\n    event: api_proto.ClientEvent | api_proto.ServerEvent,\n    maxLength: number = 30,\n  ): Record<string, unknown> {\n    const untypedEvent: Record<string, unknown> = {};\n    for (const [key, value] of Object.entries(event)) {\n      if (value !== undefined) {\n        untypedEvent[key] = value;\n      }\n    }\n\n    if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n      const truncatedData =\n        untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? '…' : '');\n      return { ...untypedEvent, audio: truncatedData };\n    }\n    if (\n      untypedEvent.delta &&\n      typeof untypedEvent.delta === 'string' &&\n      event.type === 'response.audio.delta'\n    ) {\n      const truncatedDelta =\n        untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? '…' : '');\n      return { ...untypedEvent, delta: truncatedDelta };\n    }\n    return untypedEvent;\n  }\n\n  sessionUpdate({\n    modalities = this.#opts.modalities,\n    instructions = this.#opts.instructions,\n    voice = this.#opts.voice,\n    inputAudioFormat = this.#opts.inputAudioFormat,\n    outputAudioFormat = this.#opts.outputAudioFormat,\n    inputAudioTranscription = this.#opts.inputAudioTranscription,\n    turnDetection = this.#opts.turnDetection,\n    temperature = this.#opts.temperature,\n    maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,\n    toolChoice = 'auto',\n    selectedTools = Object.keys(this.#fncCtx || {}),\n  }: {\n    modalities: ['text', 'audio'] | ['text'];\n    instructions?: string;\n    voice?: api_proto.Voice;\n    inputAudioFormat?: api_proto.AudioFormat;\n    outputAudioFormat?: api_proto.AudioFormat;\n    inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n    turnDetection?: api_proto.TurnDetectionType | null;\n    temperature?: number;\n    maxResponseOutputTokens?: number;\n    toolChoice?: api_proto.ToolChoice;\n    selectedTools?: string[];\n  }) {\n    this.#opts = {\n      modalities,\n      instructions,\n      voice,\n      inputAudioFormat,\n      outputAudioFormat,\n      inputAudioTranscription,\n      turnDetection,\n      temperature,\n      maxResponseOutputTokens,\n      model: this.#opts.model,\n      apiKey: this.#opts.apiKey,\n      baseURL: this.#opts.baseURL,\n      isAzure: this.#opts.isAzure,\n      apiVersion: this.#opts.apiVersion,\n      entraToken: this.#opts.entraToken,\n    };\n\n    const tools = this.#fncCtx\n      ? Object.entries(this.#fncCtx)\n          .filter(([name]) => selectedTools.includes(name))\n          .map(([name, func]) => ({\n            type: 'function' as const,\n            name,\n            description: func.description,\n            parameters:\n              // don't format parameters if they are raw openai params\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          }))\n      : [];\n\n    const sessionUpdateEvent: api_proto.SessionUpdateEvent = {\n      type: 'session.update',\n      session: {\n        modalities: this.#opts.modalities,\n        instructions: this.#opts.instructions,\n        voice: this.#opts.voice,\n        input_audio_format: this.#opts.inputAudioFormat,\n        output_audio_format: this.#opts.outputAudioFormat,\n        input_audio_transcription: this.#opts.inputAudioTranscription,\n        turn_detection: this.#opts.turnDetection,\n        temperature: this.#opts.temperature,\n        max_response_output_tokens:\n          this.#opts.maxResponseOutputTokens === Infinity\n            ? 'inf'\n            : this.#opts.maxResponseOutputTokens,\n        tools,\n        tool_choice: toolChoice,\n      },\n    };\n\n    if (this.#opts.isAzure && this.#opts.maxResponseOutputTokens === Infinity) {\n      // microsoft doesn't support inf for max_response_output_tokens, but accepts no args\n      sessionUpdateEvent.session.max_response_output_tokens = undefined;\n    }\n\n    this.queueMsg(sessionUpdateEvent);\n  }\n\n  /** Create an empty audio message with the given duration. */\n  #createEmptyUserAudioMessage(duration: number): llm.ChatMessage {\n    const samples = duration * api_proto.SAMPLE_RATE;\n    return new llm.ChatMessage({\n      role: llm.ChatRole.USER,\n      content: {\n        frame: new AudioFrame(\n          new Int16Array(samples * api_proto.NUM_CHANNELS),\n          api_proto.SAMPLE_RATE,\n          api_proto.NUM_CHANNELS,\n          samples,\n        ),\n      },\n    });\n  }\n\n  /**\n   * Try to recover from a text response to audio mode.\n   *\n   * @remarks\n   * Sometimes the OpenAI Realtime API returns text instead of audio responses.\n   * This method tries to recover from this by requesting a new response after deleting the text\n   * response and creating an empty user audio message.\n   */\n  recoverFromTextResponse(itemId: string) {\n    if (itemId) {\n      this.conversation.item.delete(itemId);\n    }\n    this.conversation.item.create(this.#createEmptyUserAudioMessage(1));\n    this.response.create();\n  }\n\n  #start(): Promise<void> {\n    return new Promise(async (resolve, reject) => {\n      const headers: Record<string, string> = {\n        'User-Agent': 'LiveKit-Agents-JS',\n      };\n      if (this.#opts.isAzure) {\n        // Microsoft API has two ways of authentication\n        // 1. Entra token set as `Bearer` token\n        // 2. API key set as `api_key` header (also accepts query string)\n        if (this.#opts.entraToken) {\n          headers.Authorization = `Bearer ${this.#opts.entraToken}`;\n        } else if (this.#opts.apiKey) {\n          headers['api-key'] = this.#opts.apiKey;\n        } else {\n          reject(new Error('Microsoft API key or entraToken is required'));\n          return;\n        }\n      } else {\n        headers.Authorization = `Bearer ${this.#opts.apiKey}`;\n        headers['OpenAI-Beta'] = 'realtime=v1';\n      }\n      const url = new URL([this.#opts.baseURL, 'realtime'].join('/'));\n      if (url.protocol === 'https:') {\n        url.protocol = 'wss:';\n      }\n\n      // Construct query parameters\n      const queryParams: Record<string, string> = {};\n      if (this.#opts.isAzure) {\n        queryParams['api-version'] = this.#opts.apiVersion ?? '2024-10-01-preview';\n        queryParams['deployment'] = this.#opts.model;\n      } else {\n        queryParams['model'] = this.#opts.model;\n      }\n\n      for (const [key, value] of Object.entries(queryParams)) {\n        url.searchParams.set(key, value);\n      }\n\n      console.debug('Connecting to OpenAI Realtime API at ', url.toString());\n      this.#ws = new WebSocket(url.toString(), {\n        headers: headers,\n      });\n\n      this.#ws.onerror = (error) => {\n        reject(new Error('OpenAI Realtime WebSocket error: ' + error.message));\n      };\n\n      await once(this.#ws, 'open');\n      this.#closing = false;\n\n      this.#ws.onmessage = (message) => {\n        const event: api_proto.ServerEvent = JSON.parse(message.data as string);\n        this.#logger.debug(`<- ${JSON.stringify(this.#loggableEvent(event))}`);\n        switch (event.type) {\n          case 'error':\n            this.#handleError(event);\n            break;\n          case 'session.created':\n            this.#handleSessionCreated(event);\n            break;\n          case 'session.updated':\n            this.#handleSessionUpdated(event);\n            break;\n          case 'conversation.created':\n            this.#handleConversationCreated(event);\n            break;\n          case 'input_audio_buffer.committed':\n            this.#handleInputAudioBufferCommitted(event);\n            break;\n          case 'input_audio_buffer.cleared':\n            this.#handleInputAudioBufferCleared(event);\n            break;\n          case 'input_audio_buffer.speech_started':\n            this.#handleInputAudioBufferSpeechStarted(event);\n            break;\n          case 'input_audio_buffer.speech_stopped':\n            this.#handleInputAudioBufferSpeechStopped(event);\n            break;\n          case 'conversation.item.created':\n            this.#handleConversationItemCreated(event);\n            break;\n          case 'conversation.item.input_audio_transcription.completed':\n            this.#handleConversationItemInputAudioTranscriptionCompleted(event);\n            break;\n          case 'conversation.item.input_audio_transcription.failed':\n            this.#handleConversationItemInputAudioTranscriptionFailed(event);\n            break;\n          case 'conversation.item.truncated':\n            this.#handleConversationItemTruncated(event);\n            break;\n          case 'conversation.item.deleted':\n            this.#handleConversationItemDeleted(event);\n            break;\n          case 'response.created':\n            this.#handleResponseCreated(event);\n            break;\n          case 'response.done':\n            this.#handleResponseDone(event);\n            break;\n          case 'response.output_item.added':\n            this.#handleResponseOutputItemAdded(event);\n            break;\n          case 'response.output_item.done':\n            this.#handleResponseOutputItemDone(event);\n            break;\n          case 'response.content_part.added':\n            this.#handleResponseContentPartAdded(event);\n            break;\n          case 'response.content_part.done':\n            this.#handleResponseContentPartDone(event);\n            break;\n          case 'response.text.delta':\n            this.#handleResponseTextDelta(event);\n            break;\n          case 'response.text.done':\n            this.#handleResponseTextDone(event);\n            break;\n          case 'response.audio_transcript.delta':\n            this.#handleResponseAudioTranscriptDelta(event);\n            break;\n          case 'response.audio_transcript.done':\n            this.#handleResponseAudioTranscriptDone(event);\n            break;\n          case 'response.audio.delta':\n            this.#handleResponseAudioDelta(event);\n            break;\n          case 'response.audio.done':\n            this.#handleResponseAudioDone(event);\n            break;\n          case 'response.function_call_arguments.delta':\n            this.#handleResponseFunctionCallArgumentsDelta(event);\n            break;\n          case 'response.function_call_arguments.done':\n            this.#handleResponseFunctionCallArgumentsDone(event);\n            break;\n          case 'rate_limits.updated':\n            this.#handleRateLimitsUpdated(event);\n            break;\n        }\n      };\n\n      const sendTask = async () => {\n        while (this.#ws && !this.#closing && this.#ws.readyState === WebSocket.OPEN) {\n          try {\n            const event = await this.#sendQueue.get();\n            if (event.type !== 'input_audio_buffer.append') {\n              this.#logger.debug(`-> ${JSON.stringify(this.#loggableEvent(event))}`);\n            }\n            this.#ws.send(JSON.stringify(event));\n          } catch (error) {\n            this.#logger.error('Error sending event:', error);\n          }\n        }\n      };\n\n      sendTask();\n\n      this.#ws.onclose = () => {\n        if (this.#expiresAt && Date.now() >= this.#expiresAt * 1000) {\n          this.#closing = true;\n        }\n        if (!this.#closing) {\n          reject(new Error('OpenAI Realtime connection closed unexpectedly'));\n        }\n        this.#ws = null;\n        resolve();\n      };\n    });\n  }\n\n  async close() {\n    if (!this.#ws) return;\n    this.#closing = true;\n    this.#ws.close();\n    await this.#task;\n  }\n\n  #getContent(ptr: ContentPtr): RealtimeContent {\n    const response = this.#pendingResponses[ptr.response_id];\n    const output = response!.output[ptr.output_index];\n    const content = output!.content[ptr.content_index]!;\n    return content;\n  }\n\n  #handleError(event: api_proto.ErrorEvent): void {\n    this.#logger.error(`OpenAI Realtime error ${JSON.stringify(event.error)}`);\n  }\n\n  #handleSessionCreated(event: api_proto.SessionCreatedEvent): void {\n    this.#sessionId = event.session.id;\n    this.#expiresAt = event.session.expires_at;\n    this.#logger = this.#logger.child({ sessionId: this.#sessionId });\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleSessionUpdated(event: api_proto.SessionUpdatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationCreated(event: api_proto.ConversationCreatedEvent): void {}\n\n  #handleInputAudioBufferCommitted(event: api_proto.InputAudioBufferCommittedEvent): void {\n    this.emit('input_speech_committed', {\n      itemId: event.item_id,\n    } as InputSpeechCommitted);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleInputAudioBufferCleared(event: api_proto.InputAudioBufferClearedEvent): void {}\n\n  #handleInputAudioBufferSpeechStarted(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStartedEvent,\n  ): void {\n    this.emit('input_speech_started', {\n      itemId: event.item_id,\n    } as InputSpeechStarted);\n  }\n\n  #handleInputAudioBufferSpeechStopped(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.InputAudioBufferSpeechStoppedEvent,\n  ): void {\n    this.emit('input_speech_stopped');\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {}\n\n  #handleConversationItemInputAudioTranscriptionCompleted(\n    event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n  ): void {\n    const transcript = event.transcript;\n    this.emit('input_speech_transcription_completed', {\n      itemId: event.item_id,\n      transcript: transcript,\n    } as InputSpeechTranscriptionCompleted);\n  }\n\n  #handleConversationItemInputAudioTranscriptionFailed(\n    event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n  ): void {\n    const error = event.error;\n    this.#logger.error(`OpenAI Realtime failed to transcribe input audio: ${error.message}`);\n    this.emit('input_speech_transcription_failed', {\n      itemId: event.item_id,\n      message: error.message,\n    } as InputSpeechTranscriptionFailed);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemTruncated(event: api_proto.ConversationItemTruncatedEvent): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {}\n\n  #handleResponseCreated(responseCreated: api_proto.ResponseCreatedEvent): void {\n    const response = responseCreated.response;\n    const doneFut = new Future();\n    const newResponse: RealtimeResponse = {\n      id: response.id,\n      status: response.status,\n      statusDetails: response.status_details,\n      usage: null,\n      output: [],\n      doneFut: doneFut,\n      createdTimestamp: Date.now(),\n    };\n    this.#pendingResponses[newResponse.id] = newResponse;\n    this.emit('response_created', newResponse);\n  }\n\n  #handleResponseDone(event: api_proto.ResponseDoneEvent): void {\n    const responseData = event.response;\n    const responseId = responseData.id;\n    const response = this.#pendingResponses[responseId]!;\n    response.status = responseData.status;\n    response.statusDetails = responseData.status_details;\n    response.usage = responseData.usage ?? null;\n    this.#pendingResponses[responseId] = response;\n    response.doneFut.resolve();\n\n    let metricsError: Error | undefined;\n    let cancelled = false;\n    switch (response.status) {\n      case 'failed': {\n        if (response.statusDetails.type !== 'failed') break;\n        const err = response.statusDetails.error;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          code: err?.code,\n          message: err?.message,\n        });\n        this.#logger\n          .child({ code: err?.code, error: err?.message })\n          .error('response generation failed');\n        break;\n      }\n      case 'incomplete': {\n        if (response.statusDetails.type !== 'incomplete') break;\n        const reason = response.statusDetails.reason;\n        metricsError = new metrics.MultimodalLLMError({\n          type: response.statusDetails.type,\n          reason,\n        });\n        this.#logger.child({ reason }).error('response generation incomplete');\n        break;\n      }\n      case 'cancelled': {\n        cancelled = true;\n        break;\n      }\n    }\n    this.emit('response_done', response);\n\n    let ttft: number | undefined;\n    if (response.firstTokenTimestamp) {\n      ttft = response.firstTokenTimestamp - response.createdTimestamp;\n    }\n    const duration = Date.now() - response.createdTimestamp;\n\n    const usage = response.usage;\n    const metric: metrics.MultimodalLLMMetrics = {\n      timestamp: response.createdTimestamp,\n      requestId: response.id,\n      ttft: ttft!,\n      duration,\n      cancelled,\n      label: this.constructor.name,\n      completionTokens: usage?.output_tokens || 0,\n      promptTokens: usage?.input_tokens || 0,\n      totalTokens: usage?.total_tokens || 0,\n      tokensPerSecond: ((usage?.output_tokens || 0) / duration) * 1000,\n      error: metricsError,\n      inputTokenDetails: {\n        cachedTokens: usage?.input_token_details.cached_tokens || 0,\n        textTokens: usage?.input_token_details.text_tokens || 0,\n        audioTokens: usage?.input_token_details.audio_tokens || 0,\n      },\n      outputTokenDetails: {\n        textTokens: usage?.output_token_details.text_tokens || 0,\n        audioTokens: usage?.output_token_details.audio_tokens || 0,\n      },\n    };\n    this.emit('metrics_collected', metric);\n  }\n\n  #handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const itemData = event.item;\n\n    if (itemData.type !== 'message' && itemData.type !== 'function_call') {\n      throw new Error(`Unexpected item type: ${itemData.type}`);\n    }\n\n    let role: api_proto.Role;\n    if (itemData.type === 'function_call') {\n      role = 'assistant'; // function_call doesn't have a role field, defaulting it to assistant\n    } else {\n      role = itemData.role;\n    }\n\n    const newOutput: RealtimeOutput = {\n      responseId: responseId,\n      itemId: itemData.id,\n      outputIndex: event.output_index,\n      type: itemData.type,\n      role: role,\n      content: [],\n      doneFut: new Future(),\n    };\n    response?.output.push(newOutput);\n    this.emit('response_output_added', newOutput);\n  }\n\n  #handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    if (output?.type === 'function_call') {\n      if (!this.#fncCtx) {\n        this.#logger.error('function call received but no fncCtx is available');\n        return;\n      }\n\n      // parse the arguments and call the function inside the fnc_ctx\n      const item = event.item;\n      if (item.type !== 'function_call') {\n        throw new Error('Expected function_call item');\n      }\n      const func = this.#fncCtx[item.name];\n      if (!func) {\n        this.#logger.error(`no function with name ${item.name} in fncCtx`);\n        return;\n      }\n\n      this.emit('function_call_started', {\n        callId: item.call_id,\n      });\n\n      const parsedArgs = JSON.parse(item.arguments);\n\n      this.#logger.debug(\n        `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`,\n      );\n\n      func.execute(parsedArgs).then(\n        (content) => {\n          this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);\n          this.emit('function_call_completed', {\n            callId: item.call_id,\n          });\n          this.conversation.item.create(\n            llm.ChatMessage.createToolFromFunctionResult({\n              name: item.name,\n              toolCallId: item.call_id,\n              result: content,\n            }),\n            output.itemId,\n          );\n          this.response.create();\n        },\n        (error) => {\n          this.#logger.error(`[Function Call ${item.call_id}] ${item.name} failed with ${error}`);\n          // TODO: send it back up as failed?\n          this.emit('function_call_failed', {\n            callId: item.call_id,\n          });\n        },\n      );\n    }\n\n    output?.doneFut.resolve();\n    this.emit('response_output_done', output);\n  }\n\n  #handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n    const responseId = event.response_id;\n    const response = this.#pendingResponses[responseId];\n    const outputIndex = event.output_index;\n    const output = response!.output[outputIndex];\n\n    const textStream = new AsyncIterableQueue<string>();\n    const audioStream = new AsyncIterableQueue<AudioFrame>();\n\n    const newContent: RealtimeContent = {\n      responseId: responseId,\n      itemId: event.item_id,\n      outputIndex: outputIndex,\n      contentIndex: event.content_index,\n      text: '',\n      audio: [],\n      textStream: textStream,\n      audioStream: audioStream,\n      toolCalls: [],\n      contentType: event.part.type,\n    };\n    output?.content.push(newContent);\n    response!.firstTokenTimestamp = Date.now();\n    this.emit('response_content_added', newContent);\n  }\n\n  #handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n    const content = this.#getContent(event);\n    this.emit('response_content_done', content);\n  }\n\n  #handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n    this.emit('response_text_delta', event);\n  }\n\n  #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {\n    const content = this.#getContent(event);\n    content.text = event.text;\n    this.emit('response_text_done', event);\n  }\n\n  #handleResponseAudioTranscriptDelta(event: api_proto.ResponseAudioTranscriptDeltaEvent): void {\n    const content = this.#getContent(event);\n    const transcript = event.delta;\n    content.text += transcript;\n\n    content.textStream.put(transcript);\n  }\n\n  #handleResponseAudioTranscriptDone(event: api_proto.ResponseAudioTranscriptDoneEvent): void {\n    const content = this.#getContent(event);\n    content.textStream.close();\n  }\n\n  #handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n    const content = this.#getContent(event);\n    const data = Buffer.from(event.delta, 'base64');\n    const audio = new AudioFrame(\n      new Int16Array(data.buffer),\n      api_proto.SAMPLE_RATE,\n      api_proto.NUM_CHANNELS,\n      data.length / 2,\n    );\n    content.audio.push(audio);\n\n    content.audioStream.put(audio);\n  }\n\n  #handleResponseAudioDone(event: api_proto.ResponseAudioDoneEvent): void {\n    const content = this.#getContent(event);\n    content.audioStream.close();\n  }\n\n  #handleResponseFunctionCallArgumentsDelta(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDeltaEvent,\n  ): void {}\n\n  #handleResponseFunctionCallArgumentsDone(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    event: api_proto.ResponseFunctionCallArgumentsDoneEvent,\n  ): void {}\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  #handleRateLimitsUpdated(event: api_proto.RateLimitsUpdatedEvent): void {}\n}\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,YAAY,eAAe;AA8E3B,MAAM,iBAAiB;AAAA,EACrB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,OAAO,OAAmB;AACxB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,OAAO,OAAO,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,IACzD,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAEA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS,QAAgB,cAAsB,UAAkB;AAC/D,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,MACT,eAAe;AAAA,MACf,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,QAAgB;AACrB,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,SAA0B,gBAA+B;AAC9D,QAAI,CAAC,QAAQ,SAAS;AACpB;AAAA,IACF;AAEA,QAAI;AAEJ,QAAI,QAAQ,YAAY;AACtB,UAAI,OAAO,QAAQ,YAAY,UAAU;AACvC,cAAM,IAAI,UAAU,kCAAkC;AAAA,MACxD;AAEA,cAAQ;AAAA,QACN,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,SAAS,QAAQ;AAAA,UACjB,QAAQ,QAAQ;AAAA,QAClB;AAAA,MACF;AAAA,IACF,OAAO;AACL,UAAI,UAAU,QAAQ;AACtB,UAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;AAC3B,kBAAU,CAAC,OAAO;AAAA,MACpB;AAEA,UAAI,QAAQ,SAAS,IAAI,SAAS,MAAM;AACtC,cAAM,WAAyE,CAAC;AAChF,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,OAAO,OAAO,KAAK,YAAY,EAAE,KAAK,EAAE,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,YACxE,CAAC;AAAA,UACH;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,IAAI,SAAS,WAAW;AAClD,cAAM,WAAoC,CAAC;AAC3C,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,qDAAqD;AAAA,UACzE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,WAAW,QAAQ,SAAS,IAAI,SAAS,QAAQ;AAC/C,cAAM,WAAyC,CAAC;AAChD,mBAAW,KAAK,SAAS;AACvB,cAAI,OAAO,MAAM,UAAU;AACzB,qBAAS,KAAK;AAAA,cACZ,MAAM;AAAA,cACN,MAAM;AAAA,YACR,CAAC;AAAA,UACH;AAAA;AAAA,aAEG,CAACA,OAAyD;AACzD,qBAAQA,GAAoB,UAAU;AAAA,YACxC,GAAG,CAAC;AAAA,YACJ;AACA,iBAAK,QAAQ,KAAK,kDAAkD;AAAA,UACtE;AAAA,QACF;AAEA,gBAAQ;AAAA,UACN,MAAM;AAAA,UACN,kBAAkB;AAAA,UAClB,MAAM;AAAA,YACJ,MAAM;AAAA,YACN,MAAM;AAAA,YACN,SAAS;AAAA,UACX;AAAA,QACF;AAAA,MACF,OAAO;AACL,aAAK,QACF,MAAM,EAAE,QAAQ,CAAC,EACjB,KAAK,uDAAuD;AAC/D;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,SAAS,KAAK;AAAA,EAC9B;AACF;AAEA,MAAM,aAAa;AAAA,EACjB;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,IAAI,OAAyB;AAC3B,WAAO,IAAI,iBAAiB,KAAK,QAAQ;AAAA,EAC3C;AACF;AAEA,MAAM,SAAS;AAAA,EACb;AAAA,EAEA,YAAY,SAA0B;AACpC,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,SAAS,SAAS;AAAA,MACrB,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAQO,MAAM,sBAAsB,WAAW,cAAc;AAAA,EAC1D,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA,EAEzB;AAAA,EACA,YAA+B,CAAC;AAAA,EAEhC,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA,aAAa;AAAA,IACb,SAAS;AAAA,IACT,aAAa;AAAA,IACb,eAAe;AAAA,IACf,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,EAC5B,GAeG;AACD,WAAO,IAAI,cAAc;AAAA,MACvB,SAAS;AAAA,MACT,SAAS,IAAI,IAAI,UAAU,OAAO,EAAE,SAAS;AAAA,MAC7C,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,YAAY;AAAA,IACV,aAAa,CAAC,QAAQ,OAAO;AAAA,IAC7B,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,mBAAmB;AAAA,IACnB,oBAAoB;AAAA,IACpB,0BAA0B,EAAE,OAAO,YAAY;AAAA,IAC/C,gBAAgB,EAAE,MAAM,aAAa;AAAA,IACrC,cAAc;AAAA,IACd,0BAA0B;AAAA,IAC1B,QAAQ;AAAA,IACR,SAAS,QAAQ,IAAI,kBAAkB;AAAA,IACvC,UAAU,UAAU;AAAA;AAAA,IAEpB,UAAU;AAAA,IACV,aAAa;AAAA,IACb,aAAa;AAAA,EACf,GAgBG;AACD,UAAM;AAEN,QAAI,WAAW,IAAI;AACjB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,SAAK,eAAe;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,WAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,QAAQ;AAAA,IACN;AAAA,IACA;AAAA,IACA,aAAa,KAAK,aAAa;AAAA,IAC/B,eAAe,KAAK,aAAa;AAAA,IACjC,QAAQ,KAAK,aAAa;AAAA,IAC1B,mBAAmB,KAAK,aAAa;AAAA,IACrC,oBAAoB,KAAK,aAAa;AAAA,IACtC,0BAA0B,KAAK,aAAa;AAAA,IAC5C,gBAAgB,KAAK,aAAa;AAAA,IAClC,cAAc,KAAK,aAAa;AAAA,IAChC,0BAA0B,KAAK,aAAa;AAAA,EAC9C,GAYoB;AAClB,UAAM,OAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,aAAa;AAAA,MACzB,QAAQ,KAAK,aAAa;AAAA,MAC1B,SAAS,KAAK,aAAa;AAAA,MAC3B,SAAS,KAAK,aAAa;AAAA,MAC3B,YAAY,KAAK,aAAa;AAAA,MAC9B,YAAY,KAAK,aAAa;AAAA,IAChC;AAEA,UAAM,aAAa,IAAI,gBAAgB,MAAM;AAAA,MAC3C,SAAS,WAAW,IAAI,IAAI,YAAY;AAAA,MACxC;AAAA,IACF,CAAC;AACD,SAAK,UAAU,KAAK,UAAU;AAC9B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,QAAQ,WAAW,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC3E;AACF;AAEO,MAAM,wBAAwB,WAAW,gBAAgB;AAAA,EAC9D,WAAwC;AAAA,EACxC,UAA2C;AAAA,EAC3C;AAAA,EACA,oBAAwD,CAAC;AAAA,EACzD,aAAa;AAAA,EACb,MAAwB;AAAA,EACxB,aAA4B;AAAA,EAC5B,UAAU,IAAI;AAAA,EACd;AAAA,EACA,WAAW;AAAA,EACX,aAAa,IAAI,MAA6B;AAAA,EAE9C,YACE,MACA,EAAE,QAAQ,QAAQ,GAClB;AACA,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,UAAU;AAEf,SAAK,QAAQ,KAAK,OAAO;AAEzB,SAAK,cAAc;AAAA,MACjB,YAAY,KAAK,MAAM;AAAA,MACvB,cAAc,KAAK,MAAM;AAAA,MACzB,OAAO,KAAK,MAAM;AAAA,MAClB,kBAAkB,KAAK,MAAM;AAAA,MAC7B,mBAAmB,KAAK,MAAM;AAAA,MAC9B,yBAAyB,KAAK,MAAM;AAAA,MACpC,eAAe,KAAK,MAAM;AAAA,MAC1B,aAAa,KAAK,MAAM;AAAA,MACxB,yBAAyB,KAAK,MAAM;AAAA,MACpC,YAAY;AAAA,IACd,CAAC;AAAA,EACH;AAAA,EAEA,IAAI,UAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAA0C;AAC5C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAO,KAAsC;AAC/C,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,IAAI,eAA6B;AAC/B,WAAO,IAAI,aAAa,IAAI;AAAA,EAC9B;AAAA,EAEA,IAAI,mBAAqC;AACvC,WAAO,IAAI,iBAAiB,IAAI;AAAA,EAClC;AAAA,EAEA,IAAI,WAAqB;AACvB,WAAO,IAAI,SAAS,IAAI;AAAA,EAC1B;AAAA,EAEA,IAAI,aAAqB;AACvB,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,qBAAqB;AAAA,IACvC;AACA,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,SAAS,SAAsC;AAC7C,SAAK,WAAW,IAAI,OAAO;AAAA,EAC7B;AAAA;AAAA;AAAA,EAIA,eACE,OACA,YAAoB,IACK;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,YAAM,gBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,cAAc;AAAA,IACjD;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,YAAM,iBACJ,aAAa,MAAM,MAAM,GAAG,SAAS,KAAK,aAAa,MAAM,SAAS,YAAY,WAAM;AAC1F,aAAO,EAAE,GAAG,cAAc,OAAO,eAAe;AAAA,IAClD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc;AAAA,IACZ,aAAa,KAAK,MAAM;AAAA,IACxB,eAAe,KAAK,MAAM;AAAA,IAC1B,QAAQ,KAAK,MAAM;AAAA,IACnB,mBAAmB,KAAK,MAAM;AAAA,IAC9B,oBAAoB,KAAK,MAAM;AAAA,IAC/B,0BAA0B,KAAK,MAAM;AAAA,IACrC,gBAAgB,KAAK,MAAM;AAAA,IAC3B,cAAc,KAAK,MAAM;AAAA,IACzB,0BAA0B,KAAK,MAAM;AAAA,IACrC,aAAa;AAAA,IACb,gBAAgB,OAAO,KAAK,KAAK,WAAW,CAAC,CAAC;AAAA,EAChD,GAYG;AACD,SAAK,QAAQ;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,MAAM;AAAA,MAClB,QAAQ,KAAK,MAAM;AAAA,MACnB,SAAS,KAAK,MAAM;AAAA,MACpB,SAAS,KAAK,MAAM;AAAA,MACpB,YAAY,KAAK,MAAM;AAAA,MACvB,YAAY,KAAK,MAAM;AAAA,IACzB;AAEA,UAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EACxB,OAAO,CAAC,CAAC,IAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACtB,MAAM;AAAA,MACN;AAAA,MACA,aAAa,KAAK;AAAA,MAClB;AAAA;AAAA,QAEE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,IAAI,UAAU,KAAK,UAAU;AAAA;AAAA,IACrC,EAAE,IACJ,CAAC;AAEL,UAAM,qBAAmD;AAAA,MACvD,MAAM;AAAA,MACN,SAAS;AAAA,QACP,YAAY,KAAK,MAAM;AAAA,QACvB,cAAc,KAAK,MAAM;AAAA,QACzB,OAAO,KAAK,MAAM;AAAA,QAClB,oBAAoB,KAAK,MAAM;AAAA,QAC/B,qBAAqB,KAAK,MAAM;AAAA,QAChC,2BAA2B,KAAK,MAAM;AAAA,QACtC,gBAAgB,KAAK,MAAM;AAAA,QAC3B,aAAa,KAAK,MAAM;AAAA,QACxB,4BACE,KAAK,MAAM,4BAA4B,WACnC,QACA,KAAK,MAAM;AAAA,QACjB;AAAA,QACA,aAAa;AAAA,MACf;AAAA,IACF;AAEA,QAAI,KAAK,MAAM,WAAW,KAAK,MAAM,4BAA4B,UAAU;AAEzE,yBAAmB,QAAQ,6BAA6B;AAAA,IAC1D;AAEA,SAAK,SAAS,kBAAkB;AAAA,EAClC;AAAA;AAAA,EAGA,6BAA6B,UAAmC;AAC9D,UAAM,UAAU,WAAW,UAAU;AACrC,WAAO,IAAI,IAAI,YAAY;AAAA,MACzB,MAAM,IAAI,SAAS;AAAA,MACnB,SAAS;AAAA,QACP,OAAO,IAAI;AAAA,UACT,IAAI,WAAW,UAAU,UAAU,YAAY;AAAA,UAC/C,UAAU;AAAA,UACV,UAAU;AAAA,UACV;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,wBAAwB,QAAgB;AACtC,QAAI,QAAQ;AACV,WAAK,aAAa,KAAK,OAAO,MAAM;AAAA,IACtC;AACA,SAAK,aAAa,KAAK,OAAO,KAAK,6BAA6B,CAAC,CAAC;AAClE,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,SAAwB;AACtB,WAAO,IAAI,QAAQ,OAAO,SAAS,WAAW;AAC5C,YAAM,UAAkC;AAAA,QACtC,cAAc;AAAA,MAChB;AACA,UAAI,KAAK,MAAM,SAAS;AAItB,YAAI,KAAK,MAAM,YAAY;AACzB,kBAAQ,gBAAgB,UAAU,KAAK,MAAM,UAAU;AAAA,QACzD,WAAW,KAAK,MAAM,QAAQ;AAC5B,kBAAQ,SAAS,IAAI,KAAK,MAAM;AAAA,QAClC,OAAO;AACL,iBAAO,IAAI,MAAM,6CAA6C,CAAC;AAC/D;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,gBAAgB,UAAU,KAAK,MAAM,MAAM;AACnD,gBAAQ,aAAa,IAAI;AAAA,MAC3B;AACA,YAAM,MAAM,IAAI,IAAI,CAAC,KAAK,MAAM,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAC9D,UAAI,IAAI,aAAa,UAAU;AAC7B,YAAI,WAAW;AAAA,MACjB;AAGA,YAAM,cAAsC,CAAC;AAC7C,UAAI,KAAK,MAAM,SAAS;AACtB,oBAAY,aAAa,IAAI,KAAK,MAAM,cAAc;AACtD,oBAAY,YAAY,IAAI,KAAK,MAAM;AAAA,MACzC,OAAO;AACL,oBAAY,OAAO,IAAI,KAAK,MAAM;AAAA,MACpC;AAEA,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,YAAI,aAAa,IAAI,KAAK,KAAK;AAAA,MACjC;AAEA,cAAQ,MAAM,yCAAyC,IAAI,SAAS,CAAC;AACrE,WAAK,MAAM,IAAI,UAAU,IAAI,SAAS,GAAG;AAAA,QACvC;AAAA,MACF,CAAC;AAED,WAAK,IAAI,UAAU,CAAC,UAAU;AAC5B,eAAO,IAAI,MAAM,sCAAsC,MAAM,OAAO,CAAC;AAAA,MACvE;AAEA,YAAM,KAAK,KAAK,KAAK,MAAM;AAC3B,WAAK,WAAW;AAEhB,WAAK,IAAI,YAAY,CAAC,YAAY;AAChC,cAAM,QAA+B,KAAK,MAAM,QAAQ,IAAc;AACtE,aAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AACrE,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,aAAa,KAAK;AACvB;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,sBAAsB,KAAK;AAChC;AAAA,UACF,KAAK;AACH,iBAAK,2BAA2B,KAAK;AACrC;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,qCAAqC,KAAK;AAC/C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,wDAAwD,KAAK;AAClE;AAAA,UACF,KAAK;AACH,iBAAK,qDAAqD,KAAK;AAC/D;AAAA,UACF,KAAK;AACH,iBAAK,iCAAiC,KAAK;AAC3C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,uBAAuB,KAAK;AACjC;AAAA,UACF,KAAK;AACH,iBAAK,oBAAoB,KAAK;AAC9B;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,8BAA8B,KAAK;AACxC;AAAA,UACF,KAAK;AACH,iBAAK,gCAAgC,KAAK;AAC1C;AAAA,UACF,KAAK;AACH,iBAAK,+BAA+B,KAAK;AACzC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,wBAAwB,KAAK;AAClC;AAAA,UACF,KAAK;AACH,iBAAK,oCAAoC,KAAK;AAC9C;AAAA,UACF,KAAK;AACH,iBAAK,mCAAmC,KAAK;AAC7C;AAAA,UACF,KAAK;AACH,iBAAK,0BAA0B,KAAK;AACpC;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,UACF,KAAK;AACH,iBAAK,0CAA0C,KAAK;AACpD;AAAA,UACF,KAAK;AACH,iBAAK,yCAAyC,KAAK;AACnD;AAAA,UACF,KAAK;AACH,iBAAK,yBAAyB,KAAK;AACnC;AAAA,QACJ;AAAA,MACF;AAEA,YAAM,WAAW,YAAY;AAC3B,eAAO,KAAK,OAAO,CAAC,KAAK,YAAY,KAAK,IAAI,eAAe,UAAU,MAAM;AAC3E,cAAI;AACF,kBAAM,QAAQ,MAAM,KAAK,WAAW,IAAI;AACxC,gBAAI,MAAM,SAAS,6BAA6B;AAC9C,mBAAK,QAAQ,MAAM,MAAM,KAAK,UAAU,KAAK,eAAe,KAAK,CAAC,CAAC,EAAE;AAAA,YACvE;AACA,iBAAK,IAAI,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,UACrC,SAAS,OAAO;AACd,iBAAK,QAAQ,MAAM,wBAAwB,KAAK;AAAA,UAClD;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAET,WAAK,IAAI,UAAU,MAAM;AACvB,YAAI,KAAK,cAAc,KAAK,IAAI,KAAK,KAAK,aAAa,KAAM;AAC3D,eAAK,WAAW;AAAA,QAClB;AACA,YAAI,CAAC,KAAK,UAAU;AAClB,iBAAO,IAAI,MAAM,gDAAgD,CAAC;AAAA,QACpE;AACA,aAAK,MAAM;AACX,gBAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,CAAC,KAAK,IAAK;AACf,SAAK,WAAW;AAChB,SAAK,IAAI,MAAM;AACf,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,YAAY,KAAkC;AAC5C,UAAM,WAAW,KAAK,kBAAkB,IAAI,WAAW;AACvD,UAAM,SAAS,SAAU,OAAO,IAAI,YAAY;AAChD,UAAM,UAAU,OAAQ,QAAQ,IAAI,aAAa;AACjD,WAAO;AAAA,EACT;AAAA,EAEA,aAAa,OAAmC;AAC9C,SAAK,QAAQ,MAAM,yBAAyB,KAAK,UAAU,MAAM,KAAK,CAAC,EAAE;AAAA,EAC3E;AAAA,EAEA,sBAAsB,OAA4C;AAChE,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,aAAa,MAAM,QAAQ;AAChC,SAAK,UAAU,KAAK,QAAQ,MAAM,EAAE,WAAW,KAAK,WAAW,CAAC;AAAA,EAClE;AAAA;AAAA,EAGA,sBAAsB,OAA4C;AAAA,EAAC;AAAA;AAAA,EAGnE,2BAA2B,OAAiD;AAAA,EAAC;AAAA,EAE7E,iCAAiC,OAAuD;AACtF,SAAK,KAAK,0BAA0B;AAAA,MAClC,QAAQ,MAAM;AAAA,IAChB,CAAyB;AAAA,EAC3B;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,qCAEE,OACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,QAAQ,MAAM;AAAA,IAChB,CAAuB;AAAA,EACzB;AAAA,EAEA,qCAEE,OACM;AACN,SAAK,KAAK,sBAAsB;AAAA,EAClC;AAAA;AAAA,EAGA,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,wDACE,OACM;AACN,UAAM,aAAa,MAAM;AACzB,SAAK,KAAK,wCAAwC;AAAA,MAChD,QAAQ,MAAM;AAAA,MACd;AAAA,IACF,CAAsC;AAAA,EACxC;AAAA,EAEA,qDACE,OACM;AACN,UAAM,QAAQ,MAAM;AACpB,SAAK,QAAQ,MAAM,qDAAqD,MAAM,OAAO,EAAE;AACvF,SAAK,KAAK,qCAAqC;AAAA,MAC7C,QAAQ,MAAM;AAAA,MACd,SAAS,MAAM;AAAA,IACjB,CAAmC;AAAA,EACrC;AAAA;AAAA,EAGA,iCAAiC,OAAuD;AAAA,EAAC;AAAA;AAAA,EAGzF,+BAA+B,OAAqD;AAAA,EAAC;AAAA,EAErF,uBAAuB,iBAAuD;AAC5E,UAAM,WAAW,gBAAgB;AACjC,UAAM,UAAU,IAAI,OAAO;AAC3B,UAAM,cAAgC;AAAA,MACpC,IAAI,SAAS;AAAA,MACb,QAAQ,SAAS;AAAA,MACjB,eAAe,SAAS;AAAA,MACxB,OAAO;AAAA,MACP,QAAQ,CAAC;AAAA,MACT;AAAA,MACA,kBAAkB,KAAK,IAAI;AAAA,IAC7B;AACA,SAAK,kBAAkB,YAAY,EAAE,IAAI;AACzC,SAAK,KAAK,oBAAoB,WAAW;AAAA,EAC3C;AAAA,EAEA,oBAAoB,OAA0C;AAC5D,UAAM,eAAe,MAAM;AAC3B,UAAM,aAAa,aAAa;AAChC,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,aAAS,SAAS,aAAa;AAC/B,aAAS,gBAAgB,aAAa;AACtC,aAAS,QAAQ,aAAa,SAAS;AACvC,SAAK,kBAAkB,UAAU,IAAI;AACrC,aAAS,QAAQ,QAAQ;AAEzB,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,SAAS,QAAQ;AAAA,MACvB,KAAK,UAAU;AACb,YAAI,SAAS,cAAc,SAAS,SAAU;AAC9C,cAAM,MAAM,SAAS,cAAc;AACnC,uBAAe,IAAI,QAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B,MAAM,2BAAK;AAAA,UACX,SAAS,2BAAK;AAAA,QAChB,CAAC;AACD,aAAK,QACF,MAAM,EAAE,MAAM,2BAAK,MAAM,OAAO,2BAAK,QAAQ,CAAC,EAC9C,MAAM,4BAA4B;AACrC;AAAA,MACF;AAAA,MACA,KAAK,cAAc;AACjB,YAAI,SAAS,cAAc,SAAS,aAAc;AAClD,cAAM,SAAS,SAAS,cAAc;AACtC,uBAAe,IAAI,QAAQ,mBAAmB;AAAA,UAC5C,MAAM,SAAS,cAAc;AAAA,UAC7B;AAAA,QACF,CAAC;AACD,aAAK,QAAQ,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,gCAAgC;AACrE;AAAA,MACF;AAAA,MACA,KAAK,aAAa;AAChB,oBAAY;AACZ;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAK,iBAAiB,QAAQ;AAEnC,QAAI;AACJ,QAAI,SAAS,qBAAqB;AAChC,aAAO,SAAS,sBAAsB,SAAS;AAAA,IACjD;AACA,UAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AAEvC,UAAM,QAAQ,SAAS;AACvB,UAAM,SAAuC;AAAA,MAC3C,WAAW,SAAS;AAAA,MACpB,WAAW,SAAS;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,KAAK,YAAY;AAAA,MACxB,mBAAkB,+BAAO,kBAAiB;AAAA,MAC1C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,mBAAmB,+BAAO,kBAAiB,KAAK,WAAY;AAAA,MAC5D,OAAO;AAAA,MACP,mBAAmB;AAAA,QACjB,eAAc,+BAAO,oBAAoB,kBAAiB;AAAA,QAC1D,aAAY,+BAAO,oBAAoB,gBAAe;AAAA,QACtD,cAAa,+BAAO,oBAAoB,iBAAgB;AAAA,MAC1D;AAAA,MACA,oBAAoB;AAAA,QAClB,aAAY,+BAAO,qBAAqB,gBAAe;AAAA,QACvD,cAAa,+BAAO,qBAAqB,iBAAgB;AAAA,MAC3D;AAAA,IACF;AACA,SAAK,KAAK,qBAAqB,MAAM;AAAA,EACvC;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,WAAW,MAAM;AAEvB,QAAI,SAAS,SAAS,aAAa,SAAS,SAAS,iBAAiB;AACpE,YAAM,IAAI,MAAM,yBAAyB,SAAS,IAAI,EAAE;AAAA,IAC1D;AAEA,QAAI;AACJ,QAAI,SAAS,SAAS,iBAAiB;AACrC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,SAAS;AAAA,IAClB;AAEA,UAAM,YAA4B;AAAA,MAChC;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,aAAa,MAAM;AAAA,MACnB,MAAM,SAAS;AAAA,MACf;AAAA,MACA,SAAS,CAAC;AAAA,MACV,SAAS,IAAI,OAAO;AAAA,IACtB;AACA,yCAAU,OAAO,KAAK;AACtB,SAAK,KAAK,yBAAyB,SAAS;AAAA,EAC9C;AAAA,EAEA,8BAA8B,OAAoD;AAChF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,SAAI,iCAAQ,UAAS,iBAAiB;AACpC,UAAI,CAAC,KAAK,SAAS;AACjB,aAAK,QAAQ,MAAM,mDAAmD;AACtE;AAAA,MACF;AAGA,YAAM,OAAO,MAAM;AACnB,UAAI,KAAK,SAAS,iBAAiB;AACjC,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,YAAM,OAAO,KAAK,QAAQ,KAAK,IAAI;AACnC,UAAI,CAAC,MAAM;AACT,aAAK,QAAQ,MAAM,yBAAyB,KAAK,IAAI,YAAY;AACjE;AAAA,MACF;AAEA,WAAK,KAAK,yBAAyB;AAAA,QACjC,QAAQ,KAAK;AAAA,MACf,CAAC;AAED,YAAM,aAAa,KAAK,MAAM,KAAK,SAAS;AAE5C,WAAK,QAAQ;AAAA,QACX,kBAAkB,KAAK,OAAO,eAAe,KAAK,IAAI,mBAAmB,UAAU;AAAA,MACrF;AAEA,WAAK,QAAQ,UAAU,EAAE;AAAA,QACvB,CAAC,YAAY;AACX,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,aAAa,OAAO,EAAE;AACrF,eAAK,KAAK,2BAA2B;AAAA,YACnC,QAAQ,KAAK;AAAA,UACf,CAAC;AACD,eAAK,aAAa,KAAK;AAAA,YACrB,IAAI,YAAY,6BAA6B;AAAA,cAC3C,MAAM,KAAK;AAAA,cACX,YAAY,KAAK;AAAA,cACjB,QAAQ;AAAA,YACV,CAAC;AAAA,YACD,OAAO;AAAA,UACT;AACA,eAAK,SAAS,OAAO;AAAA,QACvB;AAAA,QACA,CAAC,UAAU;AACT,eAAK,QAAQ,MAAM,kBAAkB,KAAK,OAAO,KAAK,KAAK,IAAI,gBAAgB,KAAK,EAAE;AAEtF,eAAK,KAAK,wBAAwB;AAAA,YAChC,QAAQ,KAAK;AAAA,UACf,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,qCAAQ,QAAQ;AAChB,SAAK,KAAK,wBAAwB,MAAM;AAAA,EAC1C;AAAA,EAEA,gCAAgC,OAAsD;AACpF,UAAM,aAAa,MAAM;AACzB,UAAM,WAAW,KAAK,kBAAkB,UAAU;AAClD,UAAM,cAAc,MAAM;AAC1B,UAAM,SAAS,SAAU,OAAO,WAAW;AAE3C,UAAM,aAAa,IAAI,mBAA2B;AAClD,UAAM,cAAc,IAAI,mBAA+B;AAEvD,UAAM,aAA8B;AAAA,MAClC;AAAA,MACA,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,cAAc,MAAM;AAAA,MACpB,MAAM;AAAA,MACN,OAAO,CAAC;AAAA,MACR;AAAA,MACA;AAAA,MACA,WAAW,CAAC;AAAA,MACZ,aAAa,MAAM,KAAK;AAAA,IAC1B;AACA,qCAAQ,QAAQ,KAAK;AACrB,aAAU,sBAAsB,KAAK,IAAI;AACzC,SAAK,KAAK,0BAA0B,UAAU;AAAA,EAChD;AAAA,EAEA,+BAA+B,OAAqD;AAClF,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,SAAK,KAAK,yBAAyB,OAAO;AAAA,EAC5C;AAAA,EAEA,yBAAyB,OAA+C;AACtE,SAAK,KAAK,uBAAuB,KAAK;AAAA,EACxC;AAAA,EAEA,wBAAwB,OAA8C;AACpE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,OAAO,MAAM;AACrB,SAAK,KAAK,sBAAsB,KAAK;AAAA,EACvC;AAAA,EAEA,oCAAoC,OAA0D;AAC5F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,aAAa,MAAM;AACzB,YAAQ,QAAQ;AAEhB,YAAQ,WAAW,IAAI,UAAU;AAAA,EACnC;AAAA,EAEA,mCAAmC,OAAyD;AAC1F,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,WAAW,MAAM;AAAA,EAC3B;AAAA,EAEA,0BAA0B,OAAgD;AACxE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,UAAM,OAAO,OAAO,KAAK,MAAM,OAAO,QAAQ;AAC9C,UAAM,QAAQ,IAAI;AAAA,MAChB,IAAI,WAAW,KAAK,MAAM;AAAA,MAC1B,UAAU;AAAA,MACV,UAAU;AAAA,MACV,KAAK,SAAS;AAAA,IAChB;AACA,YAAQ,MAAM,KAAK,KAAK;AAExB,YAAQ,YAAY,IAAI,KAAK;AAAA,EAC/B;AAAA,EAEA,yBAAyB,OAA+C;AACtE,UAAM,UAAU,KAAK,YAAY,KAAK;AACtC,YAAQ,YAAY,MAAM;AAAA,EAC5B;AAAA,EAEA,0CAEE,OACM;AAAA,EAAC;AAAA,EAET,yCAEE,OACM;AAAA,EAAC;AAAA;AAAA,EAGT,yBAAyB,OAA+C;AAAA,EAAC;AAC3E;","names":["c"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-openai",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "description": "OpenAI plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
@@ -25,7 +25,7 @@
     "@livekit/agents": "^x",
     "@livekit/agents-plugin-silero": "^x",
     "@livekit/agents-plugins-test": "^x",
-    "@livekit/rtc-node": "^0.13.2",
+    "@livekit/rtc-node": "^0.13.4",
     "@microsoft/api-extractor": "^7.35.0",
     "@types/ws": "^8.5.10",
     "tsup": "^8.3.5",
@@ -37,8 +37,8 @@
     "ws": "^8.16.0"
   },
   "peerDependencies": {
-    "@livekit/rtc-node": "^0.13.2",
-    "@livekit/agents": "^0.6.2x"
+    "@livekit/rtc-node": "^0.13.4",
+    "@livekit/agents": "^0.7.1x"
   },
   "scripts": {
     "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",

package/src/llm.ts

@@ -607,11 +607,7 @@ const buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {
 
   if (typeof msg.content === 'string') {
     oaiMsg.content = msg.content;
-  } else if (
-    ((c?: llm.ChatContent | llm.ChatContent[]): c is llm.ChatContent[] => {
-      return (c as llm.ChatContent[]).length !== undefined;
-    })(msg.content)
-  ) {
+  } else if (Array.isArray(msg.content)) {
     oaiMsg.content = (await Promise.all(
       msg.content.map(async (c) => {
         if (typeof c === 'string') {
@@ -628,6 +624,8 @@ const buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {
         }
       }),
     )) as OpenAI.ChatCompletionContentPart[];
+  } else if (msg.content === undefined) {
+    oaiMsg.content = '';
   }
 
   // make sure to provide when function has been called inside the context

package/src/realtime/realtime_model.ts

@@ -599,6 +599,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     temperature = this.#opts.temperature,
     maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,
     toolChoice = 'auto',
+    selectedTools = Object.keys(this.#fncCtx || {}),
   }: {
     modalities: ['text', 'audio'] | ['text'];
     instructions?: string;
@@ -610,6 +611,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     temperature?: number;
     maxResponseOutputTokens?: number;
     toolChoice?: api_proto.ToolChoice;
+    selectedTools?: string[];
   }) {
     this.#opts = {
       modalities,
@@ -630,16 +632,18 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     };
 
     const tools = this.#fncCtx
-      ? Object.entries(this.#fncCtx).map(([name, func]) => ({
-          type: 'function' as const,
-          name,
-          description: func.description,
-          parameters:
-            // don't format parameters if they are raw openai params
-            func.parameters.type == ('object' as const)
-              ? func.parameters
-              : llm.oaiParams(func.parameters),
-        }))
+      ? Object.entries(this.#fncCtx)
+          .filter(([name]) => selectedTools.includes(name))
+          .map(([name, func]) => ({
+            type: 'function' as const,
+            name,
+            description: func.description,
+            parameters:
+              // don't format parameters if they are raw openai params
+              func.parameters.type == ('object' as const)
+                ? func.parameters
+                : llm.oaiParams(func.parameters),
+          }))
       : [];
 
     const sessionUpdateEvent: api_proto.SessionUpdateEvent = {
@@ -731,7 +735,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       // Construct query parameters
       const queryParams: Record<string, string> = {};
       if (this.#opts.isAzure) {
-        queryParams['api-version'] = '2024-10-01-preview';
+        queryParams['api-version'] = this.#opts.apiVersion ?? '2024-10-01-preview';
         queryParams['deployment'] = this.#opts.model;
       } else {
         queryParams['model'] = this.#opts.model;