@livekit/agents-plugin-openai 0.7.2 → 0.7.3

package/dist/llm.cjs

@@ -278,6 +278,7 @@ class LLM extends import_agents.llm.LLM {
   }) {
     temperature = temperature || this.#opts.temperature;
     return new LLMStream(
+      this,
       this.#client,
       chatCtx,
       fncCtx,
@@ -295,8 +296,9 @@ class LLMStream extends import_agents.llm.LLMStream {
   #client;
   #logger = (0, import_agents.log)();
   #id = (0, import_node_crypto.randomUUID)();
-  constructor(client, chatCtx, fncCtx, opts, parallelToolCalls, temperature, n) {
-    super(chatCtx, fncCtx);
+  label = "openai.LLMStream";
+  constructor(llm2, client, chatCtx, fncCtx, opts, parallelToolCalls, temperature, n) {
+    super(llm2, chatCtx, fncCtx);
     this.#client = client;
     this.#run(opts, n, parallelToolCalls, temperature);
   }

package/dist/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n\n  constructor(\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (\n    ((c?: llm.ChatContent | llm.ChatContent[]): c is llm.ChatContent[] => {\n      return (c as llm.ChatContent[]).length !== undefined;\n    })(msg.content)\n  ) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAyB;AACzB,yBAA2B;AAC3B,oBAAoC;AACpC,mBAAkB;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,qBAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,0BAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,kBAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAM,+BAAW;AAAA,EAEjB,YACE,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAM,SAAS,MAAM;AACrB,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,kBAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,kBAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,kBAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,kBAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,YACG,CAAC,MAAoE;AACpE,WAAQ,EAAwB,WAAW;AAAA,EAC7C,GAAG,IAAI,OAAO,GACd;AACA,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACA,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,cAAU,aAAAC,SAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["c","sharp"]}
+{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this,\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n  label = 'openai.LLMStream';\n\n  constructor(\n    llm: LLM,\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(llm, chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (\n    ((c?: llm.ChatContent | llm.ChatContent[]): c is llm.ChatContent[] => {\n      return (c as llm.ChatContent[]).length !== undefined;\n    })(msg.content)\n  ) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAyB;AACzB,yBAA2B;AAC3B,oBAAoC;AACpC,mBAAkB;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,qBAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,0BAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,kBAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EACd,UAAM,+BAAW;AAAA,EACjB,QAAQ;AAAA,EAER,YACEA,MACA,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAMA,MAAK,SAAS,MAAM;AAC1B,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,kBAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,kBAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,kBAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,kBAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,kBAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,YACG,CAAC,MAAoE;AACpE,WAAQ,EAAwB,WAAW;AAAA,EAC7C,GAAG,IAAI,OAAO,GACd;AACA,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACC,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,cAAU,aAAAC,SAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["llm","c","sharp"]}

package/dist/llm.d.ts

@@ -190,6 +190,7 @@ export declare class LLM extends llm.LLM {
 }
 export declare class LLMStream extends llm.LLMStream {
     #private;
-    constructor(client: OpenAI, chatCtx: llm.ChatContext, fncCtx: llm.FunctionContext | undefined, opts: LLMOptions, parallelToolCalls?: boolean, temperature?: number, n?: number);
+    label: string;
+    constructor(llm: LLM, client: OpenAI, chatCtx: llm.ChatContext, fncCtx: llm.FunctionContext | undefined, opts: LLMOptions, parallelToolCalls?: boolean, temperature?: number, n?: number);
 }
 //# sourceMappingURL=llm.d.ts.map

package/dist/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,GAAG,EAAO,MAAM,iBAAiB,CAAC;AAE3C,OAAO,EAAe,MAAM,EAAE,MAAM,QAAQ,CAAC;AAE7C,OAAO,KAAK,EACV,kBAAkB,EAClB,UAAU,EACV,kBAAkB,EAClB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACd,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAYD,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAI9B;;;;;;OAMG;gBACS,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAgBzD;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,SAAS,CACd,IAAI,GAAE;QACJ,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;QAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,oBAAoB,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;KACG,GACzB,GAAG;IAaN;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CACjB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC;QACnC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,aAAa,CAAC,IAAI,GAAE,OAAO,CAAC,UAAU,CAAM,GAAG,GAAG;IAezD;;;;;;OAMG;IACH,MAAM,CAAC,OAAO,CACZ,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC;QAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,CACb,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CACjB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC;QACnC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,CACb,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN,2CAA2C;IAC3C,MAAM,CAAC,UAAU,CACf,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IASN;;;;;;OAMG;IACH,MAAM,CAAC,cAAc,CACnB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,oBAAoB,CAAC;QACrC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CACjB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC;QACnC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,UAAU,CACf,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAAC;QACjC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN,IAAI,CAAC,EACH,OAAO,EACP,MAAM,EACN,WAAW,EACX,CAAC,EACD,iBAAiB,GAClB,EAAE;QACD,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;QACzC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QACjC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QACvB,iBAAiB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;KACzC,GAAG,SAAS;CAad;AAED,qBAAa,SAAU,SAAQ,GAAG,CAAC,SAAS;;gBASxC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,GAAG,CAAC,WAAW,EACxB,MAAM,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EACvC,IAAI,EAAE,UAAU,EAChB,iBAAiB,CAAC,EAAE,OAAO,EAC3B,WAAW,CAAC,EAAE,MAAM,EACpB,CAAC,CAAC,EAAE,MAAM;CA2Jb"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,GAAG,EAAO,MAAM,iBAAiB,CAAC;AAE3C,OAAO,EAAe,MAAM,EAAE,MAAM,QAAQ,CAAC;AAE7C,OAAO,KAAK,EACV,kBAAkB,EAClB,UAAU,EACV,kBAAkB,EAClB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACd,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAYD,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAI9B;;;;;;OAMG;gBACS,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAgBzD;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,SAAS,CACd,IAAI,GAAE;QACJ,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;QAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,oBAAoB,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;KACG,GACzB,GAAG;IAaN;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CACjB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC;QACnC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,aAAa,CAAC,IAAI,GAAE,OAAO,CAAC,UAAU,CAAM,GAAG,GAAG;IAezD;;;;;;OAMG;IACH,MAAM,CAAC,OAAO,CACZ,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC;QAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,CACb,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CACjB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC;QACnC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,CACb,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN,2CAA2C;IAC3C,MAAM,CAAC,UAAU,CACf,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IASN;;;;;;OAMG;IACH,MAAM,CAAC,cAAc,CACnB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,oBAAoB,CAAC;QACrC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CACjB,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC;QACnC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAeN;;;;;;OAMG;IACH,MAAM,CAAC,UAAU,CACf,IAAI,GAAE,OAAO,CAAC;QACZ,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAAC;QACjC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAM,GACN,GAAG;IAaN,IAAI,CAAC,EACH,OAAO,EACP,MAAM,EACN,WAAW,EACX,CAAC,EACD,iBAAiB,GAClB,EAAE;QACD,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;QACzC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QACjC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QACvB,iBAAiB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;KACzC,GAAG,SAAS;CAcd;AAED,qBAAa,SAAU,SAAQ,GAAG,CAAC,SAAS;;IAO1C,KAAK,SAAsB;gBAGzB,GAAG,EAAE,GAAG,EACR,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,GAAG,CAAC,WAAW,EACxB,MAAM,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EACvC,IAAI,EAAE,UAAU,EAChB,iBAAiB,CAAC,EAAE,OAAO,EAC3B,WAAW,CAAC,EAAE,MAAM,EACpB,CAAC,CAAC,EAAE,MAAM;CA2Jb"}

package/dist/llm.js

@@ -244,6 +244,7 @@ class LLM extends llm.LLM {
   }) {
     temperature = temperature || this.#opts.temperature;
     return new LLMStream(
+      this,
       this.#client,
       chatCtx,
       fncCtx,
@@ -261,8 +262,9 @@ class LLMStream extends llm.LLMStream {
   #client;
   #logger = log();
   #id = randomUUID();
-  constructor(client, chatCtx, fncCtx, opts, parallelToolCalls, temperature, n) {
-    super(chatCtx, fncCtx);
+  label = "openai.LLMStream";
+  constructor(llm2, client, chatCtx, fncCtx, opts, parallelToolCalls, temperature, n) {
+    super(llm2, chatCtx, fncCtx);
     this.#client = client;
     this.#run(opts, n, parallelToolCalls, temperature);
   }

package/dist/llm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n\n  constructor(\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (\n    ((c?: llm.ChatContent | llm.ChatContent[]): c is llm.ChatContent[] => {\n      return (c as llm.ChatContent[]).length !== undefined;\n    })(msg.content)\n  ) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":"AAGA,SAAS,KAAK,WAAW;AACzB,SAAS,kBAAkB;AAC3B,SAAS,aAAa,cAAc;AACpC,OAAO,WAAW;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,OAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,YAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,MAAM,WAAW;AAAA,EAEjB,YACE,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAM,SAAS,MAAM;AACrB,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,IAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,IAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,IAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,YACG,CAAC,MAAoE;AACpE,WAAQ,EAAwB,WAAW;AAAA,EAC7C,GAAG,IAAI,OAAO,GACd;AACA,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACA,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,UAAU,MAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["c"]}
+{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm, log } from '@livekit/agents';\nimport { randomUUID } from 'node:crypto';\nimport { AzureOpenAI, OpenAI } from 'openai';\nimport sharp from 'sharp';\nimport type {\n  CerebrasChatModels,\n  ChatModels,\n  DeepSeekChatModels,\n  GroqChatModels,\n  OctoChatModels,\n  PerplexityChatModels,\n  TelnyxChatModels,\n  TogetherChatModels,\n  XAIChatModels,\n} from './models.js';\n\nexport interface LLMOptions {\n  model: string | ChatModels;\n  apiKey?: string;\n  baseURL?: string;\n  user?: string;\n  temperature?: number;\n  client?: OpenAI;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.OPENAI_API_KEY,\n};\n\nconst defaultAzureLLMOptions: LLMOptions = {\n  model: 'gpt-4o',\n  apiKey: process.env.AZURE_API_KEY,\n};\n\nexport class LLM extends llm.LLM {\n  #opts: LLMOptions;\n  #client: OpenAI;\n\n  /**\n   * Create a new instance of OpenAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environmental variable.\n   */\n  constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n    super();\n\n    this.#opts = { ...defaultLLMOptions, ...opts };\n    if (this.#opts.apiKey === undefined) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: opts.baseURL,\n        apiKey: opts.apiKey,\n      });\n  }\n\n  /**\n   * Create a new instance of OpenAI LLM with Azure.\n   *\n   * @remarks\n   * This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n   * - `apiKey` from `AZURE_OPENAI_API_KEY`\n   * - `organization` from `OPENAI_ORG_ID`\n   * - `project` from `OPENAI_PROJECT_ID`\n   * - `azureAdToken` from `AZURE_OPENAI_AD_TOKEN`\n   * - `apiVersion` from `OPENAI_API_VERSION`\n   * - `azureEndpoint` from `AZURE_OPENAI_ENDPOINT`\n   */\n  static withAzure(\n    opts: {\n      model: string | ChatModels;\n      azureEndpoint?: string;\n      azureDeployment?: string;\n      apiVersion?: string;\n      apiKey?: string;\n      azureAdToken?: string;\n      azureAdTokenProvider?: () => Promise<string>;\n      organization?: string;\n      project?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n    } = defaultAzureLLMOptions,\n  ): LLM {\n    opts = { ...defaultLLMOptions, ...opts };\n    if (opts.apiKey === undefined) {\n      throw new Error('Azure API key is required, whether as an argument or as $AZURE_API_KEY');\n    }\n\n    return new LLM({\n      temperature: opts.temperature,\n      user: opts.user,\n      client: new AzureOpenAI(opts),\n    });\n  }\n\n  /**\n   * Create a new instance of Cerebras LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Cerebras API key, either using the argument or by setting the\n   * `CEREBRAS_API_KEY` environmental variable.\n   */\n  static withCerebras(\n    opts: Partial<{\n      model: string | CerebrasChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.CEREBRAS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Cerebras API key is required, whether as an argument or as $CEREBRAS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama3.1-8b',\n      baseURL: 'https://api.cerebras.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Fireworks LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Fireworks API key, either using the argument or by setting the\n   * `FIREWORKS_API_KEY` environmental variable.\n   */\n  static withFireworks(opts: Partial<LLMOptions> = {}): LLM {\n    opts.apiKey = opts.apiKey || process.env.FIREWORKS_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'Fireworks API key is required, whether as an argument or as $FIREWORKS_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'accounts/fireworks/models/llama-v3p1-70b-instruct',\n      baseURL: 'https://api.fireworks.ai/inference/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of xAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your xAI API key, either using the argument or by setting the\n   * `XAI_API_KEY` environmental variable.\n   */\n  static withXAI(\n    opts: Partial<{\n      model: string | XAIChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.XAI_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('xAI API key is required, whether as an argument or as $XAI_API_KEY');\n    }\n\n    return new LLM({\n      model: 'grok-2-public',\n      baseURL: 'https://api.x.ai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Groq LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Groq API key, either using the argument or by setting the\n   * `GROQ_API_KEY` environmental variable.\n   */\n  static withGroq(\n    opts: Partial<{\n      model: string | GroqChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');\n    }\n\n    return new LLM({\n      model: 'llama3-8b-8192',\n      baseURL: 'https://api.groq.com/openai/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of DeepSeek LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your DeepSeek API key, either using the argument or by setting the\n   * `DEEPSEEK_API_KEY` environmental variable.\n   */\n  static withDeepSeek(\n    opts: Partial<{\n      model: string | DeepSeekChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.DEEPSEEK_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'DeepSeek API key is required, whether as an argument or as $DEEPSEEK_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'deepseek-chat',\n      baseURL: 'https://api.deepseek.com/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of OctoAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your OctoAI API key, either using the argument or by setting the\n   * `OCTOAI_TOKEN` environmental variable.\n   */\n  static withOcto(\n    opts: Partial<{\n      model: string | OctoChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.OCTOAI_TOKEN;\n    if (opts.apiKey === undefined) {\n      throw new Error('OctoAI API key is required, whether as an argument or as $OCTOAI_TOKEN');\n    }\n\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      ...opts,\n    });\n  }\n\n  /** Create a new instance of Ollama LLM. */\n  static withOllama(\n    opts: Partial<{\n      model: string;\n      baseURL?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    return new LLM({\n      model: 'llama-2-13b-chat',\n      baseURL: 'https://text.octoai.run/v1',\n      apiKey: 'ollama',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of PerplexityAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your PerplexityAI API key, either using the argument or by setting the\n   * `PERPLEXITY_API_KEY` environmental variable.\n   */\n  static withPerplexity(\n    opts: Partial<{\n      model: string | PerplexityChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.PERPLEXITY_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'PerplexityAI API key is required, whether as an argument or as $PERPLEXITY_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'llama-3.1-sonar-small-128k-chat',\n      baseURL: 'https://api.perplexity.ai',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of TogetherAI LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your TogetherAI API key, either using the argument or by setting the\n   * `TOGETHER_API_KEY` environmental variable.\n   */\n  static withTogether(\n    opts: Partial<{\n      model: string | TogetherChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TOGETHER_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error(\n        'TogetherAI API key is required, whether as an argument or as $TOGETHER_API_KEY',\n      );\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo',\n      baseURL: 'https://api.together.xyz/v1',\n      ...opts,\n    });\n  }\n\n  /**\n   * Create a new instance of Telnyx LLM.\n   *\n   * @remarks\n   * `apiKey` must be set to your Telnyx API key, either using the argument or by setting the\n   * `TELNYX_API_KEY` environmental variable.\n   */\n  static withTelnyx(\n    opts: Partial<{\n      model: string | TelnyxChatModels;\n      apiKey?: string;\n      baseURL?: string;\n      user?: string;\n      temperature?: number;\n      client: OpenAI;\n    }> = {},\n  ): LLM {\n    opts.apiKey = opts.apiKey || process.env.TELNYX_API_KEY;\n    if (opts.apiKey === undefined) {\n      throw new Error('Telnyx API key is required, whether as an argument or as $TELNYX_API_KEY');\n    }\n\n    return new LLM({\n      model: 'meta-llama/Meta-Llama-3.1-70B-Instruct',\n      baseURL: 'https://api.telnyx.com/v2/ai',\n      ...opts,\n    });\n  }\n\n  chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: llm.ChatContext;\n    fncCtx?: llm.FunctionContext | undefined;\n    temperature?: number | undefined;\n    n?: number | undefined;\n    parallelToolCalls?: boolean | undefined;\n  }): LLMStream {\n    temperature = temperature || this.#opts.temperature;\n\n    return new LLMStream(\n      this,\n      this.#client,\n      chatCtx,\n      fncCtx,\n      this.#opts,\n      parallelToolCalls,\n      temperature,\n      n,\n    );\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  #toolCallId?: string;\n  #fncName?: string;\n  #fncRawArguments?: string;\n  #client: OpenAI;\n  #logger = log();\n  #id = randomUUID();\n  label = 'openai.LLMStream';\n\n  constructor(\n    llm: LLM,\n    client: OpenAI,\n    chatCtx: llm.ChatContext,\n    fncCtx: llm.FunctionContext | undefined,\n    opts: LLMOptions,\n    parallelToolCalls?: boolean,\n    temperature?: number,\n    n?: number,\n  ) {\n    super(llm, chatCtx, fncCtx);\n    this.#client = client;\n    this.#run(opts, n, parallelToolCalls, temperature);\n  }\n\n  async #run(opts: LLMOptions, n?: number, parallelToolCalls?: boolean, temperature?: number) {\n    const tools = this.fncCtx\n      ? Object.entries(this.fncCtx).map(([name, func]) => ({\n          type: 'function' as const,\n          function: {\n            name,\n            description: func.description,\n            // don't format parameters if they are raw openai params\n            parameters:\n              func.parameters.type == ('object' as const)\n                ? func.parameters\n                : llm.oaiParams(func.parameters),\n          },\n        }))\n      : undefined;\n\n    try {\n      const stream = await this.#client.chat.completions.create({\n        model: opts.model,\n        user: opts.user,\n        n,\n        messages: await Promise.all(\n          this.chatCtx.messages.map(async (m) => await buildMessage(m, this.#id)),\n        ),\n        temperature: temperature || opts.temperature,\n        stream_options: { include_usage: true },\n        stream: true,\n        tools,\n        parallel_tool_calls: this.fncCtx && parallelToolCalls,\n      });\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          const chatChunk = this.#parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            this.queue.put(chatChunk);\n          }\n\n          if (chunk.usage) {\n            const usage = chunk.usage;\n            this.queue.put({\n              requestId: chunk.id,\n              choices: [],\n              usage: {\n                completionTokens: usage.completion_tokens,\n                promptTokens: usage.prompt_tokens,\n                totalTokens: usage.total_tokens,\n              },\n            });\n          }\n        }\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  #parseChoice(id: string, choice: OpenAI.ChatCompletionChunk.Choice): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        let callChunk: llm.ChatChunk | undefined;\n        if (this.#toolCallId && tool.id && tool.id !== this.#toolCallId) {\n          callChunk = this.#tryBuildFunction(id, choice);\n        }\n\n        if (tool.function.name) {\n          this.#toolCallId = tool.id;\n          this.#fncName = tool.function.name;\n          this.#fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.#fncRawArguments += tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.#toolCallId\n    ) {\n      // we're done with the tool calls, run the last one\n      return this.#tryBuildFunction(id, choice);\n    }\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: { content: delta.content || undefined, role: llm.ChatRole.ASSISTANT },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n\n  #tryBuildFunction(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    if (!this.fncCtx) {\n      this.#logger.warn('oai stream tried to run function without function context');\n      return undefined;\n    }\n\n    if (!this.#toolCallId) {\n      this.#logger.warn('oai stream tried to run function but toolCallId is not set');\n      return undefined;\n    }\n\n    if (!this.#fncRawArguments || !this.#fncName) {\n      this.#logger.warn('oai stream tried to run function but rawArguments or fncName are not set');\n      return undefined;\n    }\n\n    const functionInfo = llm.oaiBuildFunctionInfo(\n      this.fncCtx,\n      this.#toolCallId,\n      this.#fncName,\n      this.#fncRawArguments,\n    );\n    this.#toolCallId = this.#fncName = this.#fncRawArguments = undefined;\n    this._functionCalls.push(functionInfo);\n\n    return {\n      requestId: id,\n      choices: [\n        {\n          delta: {\n            content: choice.delta.content || undefined,\n            role: llm.ChatRole.ASSISTANT,\n            toolCalls: this._functionCalls,\n          },\n          index: choice.index,\n        },\n      ],\n    };\n  }\n}\n\nconst buildMessage = async (msg: llm.ChatMessage, cacheKey: any) => {\n  const oaiMsg: Partial<OpenAI.ChatCompletionMessageParam> = {};\n\n  switch (msg.role) {\n    case llm.ChatRole.SYSTEM:\n      oaiMsg.role = 'system';\n      break;\n    case llm.ChatRole.USER:\n      oaiMsg.role = 'user';\n      break;\n    case llm.ChatRole.ASSISTANT:\n      oaiMsg.role = 'assistant';\n      break;\n    case llm.ChatRole.TOOL:\n      oaiMsg.role = 'tool';\n      if (oaiMsg.role === 'tool') {\n        oaiMsg.tool_call_id = msg.toolCallId;\n      }\n      break;\n  }\n\n  if (typeof msg.content === 'string') {\n    oaiMsg.content = msg.content;\n  } else if (\n    ((c?: llm.ChatContent | llm.ChatContent[]): c is llm.ChatContent[] => {\n      return (c as llm.ChatContent[]).length !== undefined;\n    })(msg.content)\n  ) {\n    oaiMsg.content = (await Promise.all(\n      msg.content.map(async (c) => {\n        if (typeof c === 'string') {\n          return { type: 'text', text: c };\n        } else if (\n          // typescript type guard for determining ChatAudio vs ChatImage\n          ((c: llm.ChatAudio | llm.ChatImage): c is llm.ChatImage => {\n            return (c as llm.ChatImage).image !== undefined;\n          })(c)\n        ) {\n          return await buildImageContent(c, cacheKey);\n        } else {\n          throw new Error('ChatAudio is not supported');\n        }\n      }),\n    )) as OpenAI.ChatCompletionContentPart[];\n  }\n\n  // make sure to provide when function has been called inside the context\n  // (+ raw_arguments)\n  if (msg.toolCalls && oaiMsg.role === 'assistant') {\n    oaiMsg.tool_calls = Object.entries(msg.toolCalls).map(([name, func]) => ({\n      id: func.toolCallId,\n      type: 'function' as const,\n      function: {\n        name: name,\n        arguments: func.rawParams,\n      },\n    }));\n  }\n\n  return oaiMsg as OpenAI.ChatCompletionMessageParam;\n};\n\nconst buildImageContent = async (image: llm.ChatImage, cacheKey: any) => {\n  if (typeof image.image === 'string') {\n    // image url\n    return {\n      type: 'image_url',\n      image_url: {\n        url: image.image,\n        detail: 'auto',\n      },\n    };\n  } else {\n    if (!image.cache[cacheKey]) {\n      // inside our internal implementation, we allow to put extra metadata to\n      // each ChatImage (avoid to reencode each time we do a chatcompletion request)\n      let encoded = sharp(image.image.data);\n\n      if (image.inferenceHeight && image.inferenceHeight) {\n        encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n      }\n\n      image.cache[cacheKey] = await encoded\n        .jpeg()\n        .toBuffer()\n        .then((buffer) => buffer.toString('utf-8'));\n    }\n\n    return {\n      type: 'image_url',\n      image_url: {\n        url: `data:image/jpeg;base64,${image.cache[cacheKey]}`,\n      },\n    };\n  }\n};\n"],"mappings":"AAGA,SAAS,KAAK,WAAW;AACzB,SAAS,kBAAkB;AAC3B,SAAS,aAAa,cAAc;AACpC,OAAO,WAAW;AAsBlB,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEA,MAAM,yBAAqC;AAAA,EACzC,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AACtB;AAEO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,OAAO;AAAA,MACT,SAAS,KAAK;AAAA,MACd,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EACL;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,OAAO,UACL,OAaI,wBACC;AACL,WAAO,EAAE,GAAG,mBAAmB,GAAG,KAAK;AACvC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,aAAa,KAAK;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,QAAQ,IAAI,YAAY,IAAI;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,cAAc,OAA4B,CAAC,GAAQ;AACxD,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,QACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,oEAAoE;AAAA,IACtF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,SACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,wEAAwE;AAAA,IAC1F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,OAAO,WACL,OAKK,CAAC,GACD;AACL,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,eACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,aACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,OAAO,WACL,OAOK,CAAC,GACD;AACL,SAAK,SAAS,KAAK,UAAU,QAAQ,IAAI;AACzC,QAAI,KAAK,WAAW,QAAW;AAC7B,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,WAAO,IAAI,IAAI;AAAA,MACb,OAAO;AAAA,MACP,SAAS;AAAA,MACT,GAAG;AAAA,IACL,CAAC;AAAA,EACH;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMc;AACZ,kBAAc,eAAe,KAAK,MAAM;AAExC,WAAO,IAAI;AAAA,MACT;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EACd,MAAM,WAAW;AAAA,EACjB,QAAQ;AAAA,EAER,YACEA,MACA,QACA,SACA,QACA,MACA,mBACA,aACA,GACA;AACA,UAAMA,MAAK,SAAS,MAAM;AAC1B,SAAK,UAAU;AACf,SAAK,KAAK,MAAM,GAAG,mBAAmB,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,KAAK,MAAkB,GAAY,mBAA6B,aAAsB;AAC1F,UAAM,QAAQ,KAAK,SACf,OAAO,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACjD,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,aAAa,KAAK;AAAA;AAAA,QAElB,YACE,KAAK,WAAW,QAAS,WACrB,KAAK,aACL,IAAI,UAAU,KAAK,UAAU;AAAA,MACrC;AAAA,IACF,EAAE,IACF;AAEJ,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,YAAY,OAAO;AAAA,QACxD,OAAO,KAAK;AAAA,QACZ,MAAM,KAAK;AAAA,QACX;AAAA,QACA,UAAU,MAAM,QAAQ;AAAA,UACtB,KAAK,QAAQ,SAAS,IAAI,OAAO,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC;AAAA,QACxE;AAAA,QACA,aAAa,eAAe,KAAK;AAAA,QACjC,gBAAgB,EAAE,eAAe,KAAK;AAAA,QACtC,QAAQ;AAAA,QACR;AAAA,QACA,qBAAqB,KAAK,UAAU;AAAA,MACtC,CAAC;AAED,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,gBAAM,YAAY,KAAK,aAAa,MAAM,IAAI,MAAM;AACpD,cAAI,WAAW;AACb,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAEA,cAAI,MAAM,OAAO;AACf,kBAAM,QAAQ,MAAM;AACpB,iBAAK,MAAM,IAAI;AAAA,cACb,WAAW,MAAM;AAAA,cACjB,SAAS,CAAC;AAAA,cACV,OAAO;AAAA,gBACL,kBAAkB,MAAM;AAAA,gBACxB,cAAc,MAAM;AAAA,gBACpB,aAAa,MAAM;AAAA,cACrB;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,aAAa,IAAY,QAAsE;AAC7F,UAAM,QAAQ,OAAO;AAErB,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,KAAK,eAAe,KAAK,MAAM,KAAK,OAAO,KAAK,aAAa;AAC/D,sBAAY,KAAK,kBAAkB,IAAI,MAAM;AAAA,QAC/C;AAEA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,cAAc,KAAK;AACxB,eAAK,WAAW,KAAK,SAAS;AAC9B,eAAK,mBAAmB,KAAK,SAAS,aAAa;AAAA,QACrD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,oBAAoB,KAAK,SAAS;AAAA,QACzC;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAEA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,aACL;AAEA,aAAO,KAAK,kBAAkB,IAAI,MAAM;AAAA,IAC1C;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO,EAAE,SAAS,MAAM,WAAW,QAAW,MAAM,IAAI,SAAS,UAAU;AAAA,UAC3E,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBACE,IACA,QAC2B;AAC3B,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,QAAQ,KAAK,2DAA2D;AAC7E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa;AACrB,WAAK,QAAQ,KAAK,4DAA4D;AAC9E,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,oBAAoB,CAAC,KAAK,UAAU;AAC5C,WAAK,QAAQ,KAAK,0EAA0E;AAC5F,aAAO;AAAA,IACT;AAEA,UAAM,eAAe,IAAI;AAAA,MACvB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AACA,SAAK,cAAc,KAAK,WAAW,KAAK,mBAAmB;AAC3D,SAAK,eAAe,KAAK,YAAY;AAErC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA,QACP;AAAA,UACE,OAAO;AAAA,YACL,SAAS,OAAO,MAAM,WAAW;AAAA,YACjC,MAAM,IAAI,SAAS;AAAA,YACnB,WAAW,KAAK;AAAA,UAClB;AAAA,UACA,OAAO,OAAO;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,eAAe,OAAO,KAAsB,aAAkB;AAClE,QAAM,SAAqD,CAAC;AAE5D,UAAQ,IAAI,MAAM;AAAA,IAChB,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd;AAAA,IACF,KAAK,IAAI,SAAS;AAChB,aAAO,OAAO;AACd,UAAI,OAAO,SAAS,QAAQ;AAC1B,eAAO,eAAe,IAAI;AAAA,MAC5B;AACA;AAAA,EACJ;AAEA,MAAI,OAAO,IAAI,YAAY,UAAU;AACnC,WAAO,UAAU,IAAI;AAAA,EACvB,YACG,CAAC,MAAoE;AACpE,WAAQ,EAAwB,WAAW;AAAA,EAC7C,GAAG,IAAI,OAAO,GACd;AACA,WAAO,UAAW,MAAM,QAAQ;AAAA,MAC9B,IAAI,QAAQ,IAAI,OAAO,MAAM;AAC3B,YAAI,OAAO,MAAM,UAAU;AACzB,iBAAO,EAAE,MAAM,QAAQ,MAAM,EAAE;AAAA,QACjC;AAAA;AAAA,WAEG,CAACC,OAAyD;AACzD,mBAAQA,GAAoB,UAAU;AAAA,UACxC,GAAG,CAAC;AAAA,UACJ;AACA,iBAAO,MAAM,kBAAkB,GAAG,QAAQ;AAAA,QAC5C,OAAO;AACL,gBAAM,IAAI,MAAM,4BAA4B;AAAA,QAC9C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAIA,MAAI,IAAI,aAAa,OAAO,SAAS,aAAa;AAChD,WAAO,aAAa,OAAO,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,MACvE,IAAI,KAAK;AAAA,MACT,MAAM;AAAA,MACN,UAAU;AAAA,QACR;AAAA,QACA,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,EAAE;AAAA,EACJ;AAEA,SAAO;AACT;AAEA,MAAM,oBAAoB,OAAO,OAAsB,aAAkB;AACvE,MAAI,OAAO,MAAM,UAAU,UAAU;AAEnC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,MAAM;AAAA,QACX,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF,OAAO;AACL,QAAI,CAAC,MAAM,MAAM,QAAQ,GAAG;AAG1B,UAAI,UAAU,MAAM,MAAM,MAAM,IAAI;AAEpC,UAAI,MAAM,mBAAmB,MAAM,iBAAiB;AAClD,kBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,MACtE;AAEA,YAAM,MAAM,QAAQ,IAAI,MAAM,QAC3B,KAAK,EACL,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,OAAO,CAAC;AAAA,IAC9C;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,QACT,KAAK,0BAA0B,MAAM,MAAM,QAAQ,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AACF;","names":["llm","c"]}

package/dist/realtime/realtime_model.cjs

@@ -711,7 +711,8 @@ class RealtimeSession extends import_agents.multimodal.RealtimeSession {
       statusDetails: response.status_details,
       usage: null,
       output: [],
-      doneFut
+      doneFut,
+      createdTimestamp: Date.now()
     };
     this.#pendingResponses[newResponse.id] = newResponse;
     this.emit("response_created", newResponse);
@@ -725,7 +726,65 @@ class RealtimeSession extends import_agents.multimodal.RealtimeSession {
     response.usage = responseData.usage ?? null;
     this.#pendingResponses[responseId] = response;
     response.doneFut.resolve();
+    let metricsError;
+    let cancelled = false;
+    switch (response.status) {
+      case "failed": {
+        if (response.statusDetails.type !== "failed") break;
+        const err = response.statusDetails.error;
+        metricsError = new import_agents.metrics.MultimodalLLMError({
+          type: response.statusDetails.type,
+          code: err == null ? void 0 : err.code,
+          message: err == null ? void 0 : err.message
+        });
+        this.#logger.child({ code: err == null ? void 0 : err.code, error: err == null ? void 0 : err.message }).error("response generation failed");
+        break;
+      }
+      case "incomplete": {
+        if (response.statusDetails.type !== "incomplete") break;
+        const reason = response.statusDetails.reason;
+        metricsError = new import_agents.metrics.MultimodalLLMError({
+          type: response.statusDetails.type,
+          reason
+        });
+        this.#logger.child({ reason }).error("response generation incomplete");
+        break;
+      }
+      case "cancelled": {
+        cancelled = true;
+        break;
+      }
+    }
     this.emit("response_done", response);
+    let ttft;
+    if (response.firstTokenTimestamp) {
+      ttft = response.firstTokenTimestamp - response.createdTimestamp;
+    }
+    const duration = Date.now() - response.createdTimestamp;
+    const usage = response.usage;
+    const metric = {
+      timestamp: response.createdTimestamp,
+      requestId: response.id,
+      ttft,
+      duration,
+      cancelled,
+      label: this.constructor.name,
+      completionTokens: (usage == null ? void 0 : usage.output_tokens) || 0,
+      promptTokens: (usage == null ? void 0 : usage.input_tokens) || 0,
+      totalTokens: (usage == null ? void 0 : usage.total_tokens) || 0,
+      tokensPerSecond: ((usage == null ? void 0 : usage.output_tokens) || 0) / duration * 1e3,
+      error: metricsError,
+      inputTokenDetails: {
+        cachedTokens: (usage == null ? void 0 : usage.input_token_details.cached_tokens) || 0,
+        textTokens: (usage == null ? void 0 : usage.input_token_details.text_tokens) || 0,
+        audioTokens: (usage == null ? void 0 : usage.input_token_details.audio_tokens) || 0
+      },
+      outputTokenDetails: {
+        textTokens: (usage == null ? void 0 : usage.output_token_details.text_tokens) || 0,
+        audioTokens: (usage == null ? void 0 : usage.output_token_details.audio_tokens) || 0
+      }
+    };
+    this.emit("metrics_collected", metric);
   }
   #handleResponseOutputItemAdded(event) {
     const responseId = event.response_id;
@@ -824,6 +883,7 @@ class RealtimeSession extends import_agents.multimodal.RealtimeSession {
       toolCalls: []
     };
     output == null ? void 0 : output.content.push(newContent);
+    response.firstTokenTimestamp = Date.now();
     this.emit("response_content_added", newContent);
   }
   #handleResponseContentPartDone(event) {