@livekit/agents 1.0.6 → 1.1.0

package/dist/inference/api_protos.js

@@ -0,0 +1,70 @@
+import { z } from "zod";
+const ttsSessionCreateEventSchema = z.object({
+  type: z.literal("session.create"),
+  sample_rate: z.string(),
+  encoding: z.string(),
+  model: z.string().optional(),
+  voice: z.string().optional(),
+  language: z.string().optional(),
+  extra: z.record(z.string(), z.unknown()),
+  transcript: z.string().optional()
+});
+const ttsInputTranscriptEventSchema = z.object({
+  type: z.literal("input_transcript"),
+  transcript: z.string()
+});
+const ttsSessionFlushEventSchema = z.object({
+  type: z.literal("session.flush")
+});
+const ttsSessionCloseEventSchema = z.object({
+  type: z.literal("session.close")
+});
+const ttsSessionCreatedEventSchema = z.object({
+  type: z.literal("session.created"),
+  session_id: z.string()
+});
+const ttsOutputAudioEventSchema = z.object({
+  type: z.literal("output_audio"),
+  audio: z.string(),
+  session_id: z.string()
+});
+const ttsDoneEventSchema = z.object({
+  type: z.literal("done"),
+  session_id: z.string()
+});
+const ttsSessionClosedEventSchema = z.object({
+  type: z.literal("session.closed"),
+  session_id: z.string()
+});
+const ttsErrorEventSchema = z.object({
+  type: z.literal("error"),
+  message: z.string(),
+  session_id: z.string()
+});
+const ttsClientEventSchema = z.discriminatedUnion("type", [
+  ttsSessionCreateEventSchema,
+  ttsInputTranscriptEventSchema,
+  ttsSessionFlushEventSchema,
+  ttsSessionCloseEventSchema
+]);
+const ttsServerEventSchema = z.discriminatedUnion("type", [
+  ttsSessionCreatedEventSchema,
+  ttsOutputAudioEventSchema,
+  ttsDoneEventSchema,
+  ttsSessionClosedEventSchema,
+  ttsErrorEventSchema
+]);
+export {
+  ttsClientEventSchema,
+  ttsDoneEventSchema,
+  ttsErrorEventSchema,
+  ttsInputTranscriptEventSchema,
+  ttsOutputAudioEventSchema,
+  ttsServerEventSchema,
+  ttsSessionCloseEventSchema,
+  ttsSessionClosedEventSchema,
+  ttsSessionCreateEventSchema,
+  ttsSessionCreatedEventSchema,
+  ttsSessionFlushEventSchema
+};
+//# sourceMappingURL=api_protos.js.map

package/dist/inference/api_protos.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/inference/api_protos.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { z } from 'zod';\n\nexport const ttsSessionCreateEventSchema = z.object({\n  type: z.literal('session.create'),\n  sample_rate: z.string(),\n  encoding: z.string(),\n  model: z.string().optional(),\n  voice: z.string().optional(),\n  language: z.string().optional(),\n  extra: z.record(z.string(), z.unknown()),\n  transcript: z.string().optional(),\n});\n\nexport const ttsInputTranscriptEventSchema = z.object({\n  type: z.literal('input_transcript'),\n  transcript: z.string(),\n});\n\nexport const ttsSessionFlushEventSchema = z.object({\n  type: z.literal('session.flush'),\n});\n\nexport const ttsSessionCloseEventSchema = z.object({\n  type: z.literal('session.close'),\n});\n\nexport const ttsSessionCreatedEventSchema = z.object({\n  type: z.literal('session.created'),\n  session_id: z.string(),\n});\n\nexport const ttsOutputAudioEventSchema = z.object({\n  type: z.literal('output_audio'),\n  audio: z.string(),\n  session_id: z.string(),\n});\n\nexport const ttsDoneEventSchema = z.object({\n  type: z.literal('done'),\n  session_id: z.string(),\n});\n\nexport const ttsSessionClosedEventSchema = z.object({\n  type: z.literal('session.closed'),\n  session_id: z.string(),\n});\n\nexport const ttsErrorEventSchema = z.object({\n  type: z.literal('error'),\n  message: z.string(),\n  session_id: z.string(),\n});\n\nexport const ttsClientEventSchema = z.discriminatedUnion('type', [\n  ttsSessionCreateEventSchema,\n  ttsInputTranscriptEventSchema,\n  ttsSessionFlushEventSchema,\n  ttsSessionCloseEventSchema,\n]);\n\nexport const ttsServerEventSchema = z.discriminatedUnion('type', [\n  ttsSessionCreatedEventSchema,\n  ttsOutputAudioEventSchema,\n  ttsDoneEventSchema,\n  ttsSessionClosedEventSchema,\n  ttsErrorEventSchema,\n]);\n\nexport type TtsSessionCreateEvent = z.infer<typeof ttsSessionCreateEventSchema>;\nexport type TtsInputTranscriptEvent = z.infer<typeof ttsInputTranscriptEventSchema>;\nexport type TtsSessionFlushEvent = z.infer<typeof ttsSessionFlushEventSchema>;\nexport type TtsSessionCloseEvent = z.infer<typeof ttsSessionCloseEventSchema>;\nexport type TtsSessionCreatedEvent = z.infer<typeof ttsSessionCreatedEventSchema>;\nexport type TtsOutputAudioEvent = z.infer<typeof ttsOutputAudioEventSchema>;\nexport type TtsDoneEvent = z.infer<typeof ttsDoneEventSchema>;\nexport type TtsSessionClosedEvent = z.infer<typeof ttsSessionClosedEventSchema>;\nexport type TtsErrorEvent = z.infer<typeof ttsErrorEventSchema>;\nexport type TtsClientEvent = z.infer<typeof ttsClientEventSchema>;\nexport type TtsServerEvent = z.infer<typeof ttsServerEventSchema>;\n"],"mappings":"AAGA,SAAS,SAAS;AAEX,MAAM,8BAA8B,EAAE,OAAO;AAAA,EAClD,MAAM,EAAE,QAAQ,gBAAgB;AAAA,EAChC,aAAa,EAAE,OAAO;AAAA,EACtB,UAAU,EAAE,OAAO;AAAA,EACnB,OAAO,EAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,OAAO,EAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,UAAU,EAAE,OAAO,EAAE,SAAS;AAAA,EAC9B,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC;AAAA,EACvC,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAEM,MAAM,gCAAgC,EAAE,OAAO;AAAA,EACpD,MAAM,EAAE,QAAQ,kBAAkB;AAAA,EAClC,YAAY,EAAE,OAAO;AACvB,CAAC;AAEM,MAAM,6BAA6B,EAAE,OAAO;AAAA,EACjD,MAAM,EAAE,QAAQ,eAAe;AACjC,CAAC;AAEM,MAAM,6BAA6B,EAAE,OAAO;AAAA,EACjD,MAAM,EAAE,QAAQ,eAAe;AACjC,CAAC;AAEM,MAAM,+BAA+B,EAAE,OAAO;AAAA,EACnD,MAAM,EAAE,QAAQ,iBAAiB;AAAA,EACjC,YAAY,EAAE,OAAO;AACvB,CAAC;AAEM,MAAM,4BAA4B,EAAE,OAAO;AAAA,EAChD,MAAM,EAAE,QAAQ,cAAc;AAAA,EAC9B,OAAO,EAAE,OAAO;AAAA,EAChB,YAAY,EAAE,OAAO;AACvB,CAAC;AAEM,MAAM,qBAAqB,EAAE,OAAO;AAAA,EACzC,MAAM,EAAE,QAAQ,MAAM;AAAA,EACtB,YAAY,EAAE,OAAO;AACvB,CAAC;AAEM,MAAM,8BAA8B,EAAE,OAAO;AAAA,EAClD,MAAM,EAAE,QAAQ,gBAAgB;AAAA,EAChC,YAAY,EAAE,OAAO;AACvB,CAAC;AAEM,MAAM,sBAAsB,EAAE,OAAO;AAAA,EAC1C,MAAM,EAAE,QAAQ,OAAO;AAAA,EACvB,SAAS,EAAE,OAAO;AAAA,EAClB,YAAY,EAAE,OAAO;AACvB,CAAC;AAEM,MAAM,uBAAuB,EAAE,mBAAmB,QAAQ;AAAA,EAC/D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,MAAM,uBAAuB,EAAE,mBAAmB,QAAQ;AAAA,EAC/D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;","names":[]}

package/dist/inference/index.cjs

@@ -0,0 +1,56 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var inference_exports = {};
+__export(inference_exports, {
+  LLM: () => import_llm.LLM,
+  LLMStream: () => import_llm.LLMStream,
+  STT: () => import_stt.STT,
+  TTS: () => import_tts.TTS,
+  llm: () => llm,
+  stt: () => stt,
+  tts: () => tts
+});
+module.exports = __toCommonJS(inference_exports);
+var llm = __toESM(require("./llm.cjs"), 1);
+var stt = __toESM(require("./stt.cjs"), 1);
+var tts = __toESM(require("./tts.cjs"), 1);
+var import_llm = require("./llm.cjs");
+var import_stt = require("./stt.cjs");
+var import_tts = require("./tts.cjs");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  LLM,
+  LLMStream,
+  STT,
+  TTS,
+  llm,
+  stt,
+  tts
+});
+//# sourceMappingURL=index.cjs.map

package/dist/inference/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/inference/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport * as llm from './llm.js';\nimport * as stt from './stt.js';\nimport * as tts from './tts.js';\n\nexport { LLM, LLMStream, type AzureModels, type LLMModels, type LLMOptions } from './llm.js';\nexport type { GatewayOptions, InferenceLLMOptions } from './llm.js';\nexport { STT, type STTLanguages, type STTModels, type STTOptions } from './stt.js';\nexport { TTS, type TTSModels, type TTSOptions } from './tts.js';\nexport { llm, stt, tts };\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,UAAqB;AACrB,UAAqB;AACrB,UAAqB;AAErB,iBAAkF;AAElF,iBAAwE;AACxE,iBAAqD;","names":[]}

package/dist/inference/index.d.cts

@@ -0,0 +1,9 @@
+import * as llm from './llm.js';
+import * as stt from './stt.js';
+import * as tts from './tts.js';
+export { LLM, LLMStream, type AzureModels, type LLMModels, type LLMOptions } from './llm.js';
+export type { GatewayOptions, InferenceLLMOptions } from './llm.js';
+export { STT, type STTLanguages, type STTModels, type STTOptions } from './stt.js';
+export { TTS, type TTSModels, type TTSOptions } from './tts.js';
+export { llm, stt, tts };
+//# sourceMappingURL=index.d.ts.map

package/dist/inference/index.d.ts

@@ -0,0 +1,9 @@
+import * as llm from './llm.js';
+import * as stt from './stt.js';
+import * as tts from './tts.js';
+export { LLM, LLMStream, type AzureModels, type LLMModels, type LLMOptions } from './llm.js';
+export type { GatewayOptions, InferenceLLMOptions } from './llm.js';
+export { STT, type STTLanguages, type STTModels, type STTOptions } from './stt.js';
+export { TTS, type TTSModels, type TTSOptions } from './tts.js';
+export { llm, stt, tts };
+//# sourceMappingURL=index.d.ts.map

package/dist/inference/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/inference/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,GAAG,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,WAAW,EAAE,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAC7F,YAAY,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AACpE,OAAO,EAAE,GAAG,EAAE,KAAK,YAAY,EAAE,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AACnF,OAAO,EAAE,GAAG,EAAE,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAChE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC"}

package/dist/inference/index.js

@@ -0,0 +1,16 @@
+import * as llm from "./llm.js";
+import * as stt from "./stt.js";
+import * as tts from "./tts.js";
+import { LLM, LLMStream } from "./llm.js";
+import { STT } from "./stt.js";
+import { TTS } from "./tts.js";
+export {
+  LLM,
+  LLMStream,
+  STT,
+  TTS,
+  llm,
+  stt,
+  tts
+};
+//# sourceMappingURL=index.js.map

package/dist/inference/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/inference/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport * as llm from './llm.js';\nimport * as stt from './stt.js';\nimport * as tts from './tts.js';\n\nexport { LLM, LLMStream, type AzureModels, type LLMModels, type LLMOptions } from './llm.js';\nexport type { GatewayOptions, InferenceLLMOptions } from './llm.js';\nexport { STT, type STTLanguages, type STTModels, type STTOptions } from './stt.js';\nexport { TTS, type TTSModels, type TTSOptions } from './tts.js';\nexport { llm, stt, tts };\n"],"mappings":"AAGA,YAAY,SAAS;AACrB,YAAY,SAAS;AACrB,YAAY,SAAS;AAErB,SAAS,KAAK,iBAAoE;AAElF,SAAS,WAA+D;AACxE,SAAS,WAA4C;","names":[]}

package/dist/inference/llm.cjs

@@ -0,0 +1,315 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var llm_exports = {};
+__export(llm_exports, {
+  LLM: () => LLM,
+  LLMStream: () => LLMStream
+});
+module.exports = __toCommonJS(llm_exports);
+var import_openai = __toESM(require("openai"), 1);
+var import__ = require("../index.cjs");
+var llm = __toESM(require("../llm/index.cjs"), 1);
+var import_utils = require("./utils.cjs");
+const DEFAULT_BASE_URL = "https://agent-gateway.livekit.cloud/v1";
+class LLM extends llm.LLM {
+  client;
+  opts;
+  constructor(opts) {
+    super();
+    const {
+      model,
+      temperature,
+      parallelToolCalls,
+      toolChoice,
+      maxCompletionTokens,
+      baseURL,
+      apiKey,
+      apiSecret,
+      maxRetries,
+      timeout,
+      verbosity,
+      extraKwargs
+    } = opts;
+    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
+    if (!lkApiKey) {
+      throw new Error("apiKey is required: pass apiKey or set LIVEKIT_API_KEY");
+    }
+    const lkApiSecret = apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;
+    if (!lkApiSecret) {
+      throw new Error("apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET");
+    }
+    this.opts = {
+      model,
+      temperature,
+      parallelToolCalls,
+      toolChoice,
+      verbosity,
+      maxCompletionTokens,
+      baseURL: lkBaseURL,
+      apiKey: lkApiKey,
+      apiSecret: lkApiSecret,
+      extraKwargs: extraKwargs || {}
+    };
+    this.client = new import_openai.default({
+      baseURL: this.opts.baseURL,
+      maxRetries: maxRetries || 0,
+      timeout: timeout || 15e3
+    });
+  }
+  label() {
+    return "inference.LLM";
+  }
+  get model() {
+    return this.opts.model;
+  }
+  chat({
+    chatCtx,
+    toolCtx,
+    connOptions = import__.DEFAULT_API_CONNECT_OPTIONS,
+    parallelToolCalls,
+    toolChoice,
+    // TODO(AJS-270): Add response_format parameter support
+    extraKwargs
+  }) {
+    let extras = { ...extraKwargs || {} };
+    if (this.opts.maxCompletionTokens !== void 0) {
+      extras.max_completion_tokens = this.opts.maxCompletionTokens;
+    }
+    if (this.opts.temperature !== void 0) {
+      extras.temperature = this.opts.temperature;
+    }
+    if (this.opts.verbosity !== void 0) {
+      extras.verbosity = this.opts.verbosity;
+    }
+    parallelToolCalls = parallelToolCalls !== void 0 ? parallelToolCalls : this.opts.parallelToolCalls;
+    if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== void 0) {
+      extras.parallel_tool_calls = parallelToolCalls;
+    }
+    toolChoice = toolChoice !== void 0 ? toolChoice : this.opts.toolChoice;
+    if (toolChoice) {
+      extras.tool_choice = toolChoice;
+    }
+    extras = { ...extras, ...this.opts.extraKwargs };
+    return new LLMStream(this, {
+      model: this.opts.model,
+      providerFmt: "openai",
+      client: this.client,
+      chatCtx,
+      toolCtx,
+      connOptions,
+      extraKwargs: extras,
+      gatewayOptions: {
+        apiKey: this.opts.apiKey,
+        apiSecret: this.opts.apiSecret
+      }
+    });
+  }
+}
+class LLMStream extends llm.LLMStream {
+  model;
+  providerFmt;
+  client;
+  extraKwargs;
+  gatewayOptions;
+  toolCallId;
+  toolIndex;
+  fncName;
+  fncRawArguments;
+  constructor(llm2, {
+    model,
+    providerFmt,
+    client,
+    chatCtx,
+    toolCtx,
+    gatewayOptions,
+    connOptions,
+    extraKwargs
+  }) {
+    super(llm2, { chatCtx, toolCtx, connOptions });
+    this.client = client;
+    this.gatewayOptions = gatewayOptions;
+    this.providerFmt = providerFmt;
+    this.extraKwargs = extraKwargs;
+    this.model = model;
+  }
+  async run() {
+    var _a;
+    let retryable = true;
+    this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = void 0;
+    try {
+      const messages = await this.chatCtx.toProviderFormat(
+        this.providerFmt
+      );
+      const tools = this.toolCtx ? Object.entries(this.toolCtx).map(([name, func]) => ({
+        type: "function",
+        function: {
+          name,
+          description: func.description,
+          parameters: llm.toJsonSchema(
+            func.parameters
+          )
+        }
+      })) : void 0;
+      const requestExtras = { ...this.extraKwargs };
+      if (!tools) {
+        delete requestExtras.tool_choice;
+      }
+      if (this.gatewayOptions) {
+        this.client.apiKey = await (0, import_utils.createAccessToken)(
+          this.gatewayOptions.apiKey,
+          this.gatewayOptions.apiSecret
+        );
+      }
+      const stream = await this.client.chat.completions.create(
+        {
+          model: this.model,
+          messages,
+          tools,
+          stream: true,
+          stream_options: { include_usage: true },
+          ...requestExtras
+        },
+        {
+          timeout: this.connOptions.timeoutMs
+        }
+      );
+      for await (const chunk of stream) {
+        for (const choice of chunk.choices) {
+          if (this.abortController.signal.aborted) {
+            break;
+          }
+          const chatChunk = this.parseChoice(chunk.id, choice);
+          if (chatChunk) {
+            retryable = false;
+            this.queue.put(chatChunk);
+          }
+        }
+        if (chunk.usage) {
+          const usage = chunk.usage;
+          retryable = false;
+          this.queue.put({
+            id: chunk.id,
+            usage: {
+              completionTokens: usage.completion_tokens,
+              promptTokens: usage.prompt_tokens,
+              promptCachedTokens: ((_a = usage.prompt_tokens_details) == null ? void 0 : _a.cached_tokens) || 0,
+              totalTokens: usage.total_tokens
+            }
+          });
+        }
+      }
+    } catch (error) {
+      if (error instanceof import_openai.default.APIConnectionTimeoutError) {
+        throw new import__.APITimeoutError({ options: { retryable } });
+      } else if (error instanceof import_openai.default.APIError) {
+        throw new import__.APIStatusError({
+          message: error.message,
+          options: {
+            statusCode: error.status,
+            body: error.error,
+            requestId: error.request_id,
+            retryable
+          }
+        });
+      } else {
+        throw new import__.APIConnectionError({
+          message: (0, import__.toError)(error).message,
+          options: { retryable }
+        });
+      }
+    } finally {
+      this.queue.close();
+    }
+  }
+  parseChoice(id, choice) {
+    const delta = choice.delta;
+    if (delta === void 0) return void 0;
+    if (delta.tool_calls) {
+      for (const tool of delta.tool_calls) {
+        if (!tool.function) {
+          continue;
+        }
+        let callChunk;
+        if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {
+          callChunk = this.createRunningToolCallChunk(id, delta);
+          this.toolCallId = this.fncName = this.fncRawArguments = void 0;
+        }
+        if (tool.function.name) {
+          this.toolIndex = tool.index;
+          this.toolCallId = tool.id;
+          this.fncName = tool.function.name;
+          this.fncRawArguments = tool.function.arguments || "";
+        } else if (tool.function.arguments) {
+          this.fncRawArguments = (this.fncRawArguments || "") + tool.function.arguments;
+        }
+        if (callChunk) {
+          return callChunk;
+        }
+      }
+    }
+    if (choice.finish_reason && ["tool_calls", "stop"].includes(choice.finish_reason) && this.toolCallId !== void 0) {
+      const callChunk = this.createRunningToolCallChunk(id, delta);
+      this.toolCallId = this.fncName = this.fncRawArguments = void 0;
+      return callChunk;
+    }
+    if (!delta.content) {
+      return void 0;
+    }
+    return {
+      id,
+      delta: {
+        role: "assistant",
+        content: delta.content
+      }
+    };
+  }
+  createRunningToolCallChunk(id, delta) {
+    return {
+      id,
+      delta: {
+        role: "assistant",
+        content: delta.content || void 0,
+        toolCalls: [
+          llm.FunctionCall.create({
+            callId: this.toolCallId || "",
+            name: this.fncName || "",
+            args: this.fncRawArguments || ""
+          })
+        ]
+      }
+    };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  LLM,
+  LLMStream
+});
+//# sourceMappingURL=llm.cjs.map

package/dist/inference/llm.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/inference/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport OpenAI from 'openai';\nimport {\n  APIConnectionError,\n  APIStatusError,\n  APITimeoutError,\n  DEFAULT_API_CONNECT_OPTIONS,\n  toError,\n} from '../index.js';\nimport * as llm from '../llm/index.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { type AnyModels, createAccessToken } from './utils.js';\n\nexport type AzureModels =\n  // | \"azure/gpt-5\"\n  // | \"azure/gpt-5-mini\"\n  // | \"azure/gpt-5-nano\"\n  'azure/gpt-4.1' | 'azure/gpt-4.1-mini' | 'azure/gpt-4.1-nano';\n// | \"azure/gpt-4o\"\n// | \"azure/gpt-4o-mini\"\n\n// https://inference-docs.cerebras.ai/models/overview\nexport type CerebrasModels =\n  // production models\n  | 'cerebras/llama3.1-8b'\n  | 'cerebras/llama-3.3-70b'\n  | 'cerebras/llama-4-scout-17b-16e-instruct'\n  | 'cerebras/gpt-oss-120b'\n  | 'cerebras/qwen-3-32b'\n  // preview models\n  | 'cerebras/llama-4-maverick-17b-128e-instruct'\n  | 'cerebras/qwen-3-235b-a22b-instruct-2507';\n\n// https://console.groq.com/docs/models\nexport type GroqModels =\n  // production models\n  | 'groq/llama-3.1-8b-instant'\n  | 'groq/llama-3.3-70b-versatile'\n  | 'groq/openai/gpt-oss-120b'\n  | 'groq/openai/gpt-oss-20b'\n  // preview models\n  | 'groq/meta-llama/llama-4-maverick-17b-128e-instruct'\n  | 'groq/meta-llama/llama-4-scout-17b-16e-instruct'\n  | 'groq/qwen/qwen3-32b';\n\n// https://www.baseten.co/library/tag/llms\nexport type BasetenModels =\n  | 'baseten/deepseek-ai/DeepSeek-V3-0324'\n  | 'baseten/meta-llama/Llama-4-Scout-17B-16E-Instruct'\n  | 'baseten/meta-llama/Llama-4-Maverick-17B-128E-Instruct'\n  | 'baseten/moonshotai/Kimi-K2-Instruct'\n  | 'baseten/openai/gpt-oss-120b'\n  | 'baseten/Qwen/Qwen3-235B-A22B-Instruct-2507';\n\nexport interface AzureOptions {\n  top_p?: number;\n}\n\nexport interface CerebrasOptions {\n  top_p?: number;\n}\n\nexport interface GroqOptions {\n  top_p?: number;\n}\n\nexport interface BasetenOptions {\n  top_p?: number;\n}\n\nexport type LLMModels = AzureModels | CerebrasModels | GroqModels | BasetenModels | AnyModels;\n\nexport type LLMOptions<T extends LLMModels> = T extends AzureModels\n  ? AzureOptions\n  : T extends CerebrasModels\n    ? CerebrasOptions\n    : T extends GroqOptions\n      ? GroqOptions\n      : T extends BasetenOptions\n        ? BasetenOptions\n        : Record<string, unknown>;\n\nexport type Verbosity = 'low' | 'medium' | 'high';\nconst DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';\n\nexport interface InferenceLLMOptions<TModel extends LLMModels> {\n  model: TModel;\n  temperature?: number;\n  parallelToolCalls?: boolean;\n  toolChoice?: llm.ToolChoice;\n  maxCompletionTokens?: number;\n  baseURL: string;\n  apiKey: string;\n  apiSecret: string;\n  verbosity?: Verbosity;\n  extraKwargs: Record<string, any>; // eslint-disable-line @typescript-eslint/no-explicit-any\n}\n\nexport interface GatewayOptions {\n  apiKey: string;\n  apiSecret: string;\n}\n\nexport class LLM<TModel extends LLMModels> extends llm.LLM {\n  private client: OpenAI;\n  private opts: InferenceLLMOptions<TModel>;\n\n  constructor(opts: {\n    model: TModel;\n    temperature?: number;\n    parallelToolCalls?: boolean;\n    toolChoice?: llm.ToolChoice;\n    maxCompletionTokens?: number;\n    baseURL?: string;\n    apiKey?: string;\n    apiSecret?: string;\n    maxRetries?: number;\n    timeout?: number;\n    verbosity?: Verbosity;\n    extraKwargs?: LLMOptions<TModel>;\n  }) {\n    super();\n\n    const {\n      model,\n      temperature,\n      parallelToolCalls,\n      toolChoice,\n      maxCompletionTokens,\n      baseURL,\n      apiKey,\n      apiSecret,\n      maxRetries,\n      timeout,\n      verbosity,\n      extraKwargs,\n    } = opts;\n\n    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;\n    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;\n    if (!lkApiKey) {\n      throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');\n    }\n\n    const lkApiSecret =\n      apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;\n    if (!lkApiSecret) {\n      throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');\n    }\n\n    this.opts = {\n      model,\n      temperature,\n      parallelToolCalls,\n      toolChoice,\n      verbosity,\n      maxCompletionTokens,\n      baseURL: lkBaseURL,\n      apiKey: lkApiKey,\n      apiSecret: lkApiSecret,\n      extraKwargs: extraKwargs || {},\n    };\n\n    this.client = new OpenAI({\n      baseURL: this.opts.baseURL,\n      maxRetries: maxRetries || 0,\n      timeout: timeout || 15000,\n    });\n  }\n\n  label(): string {\n    return 'inference.LLM';\n  }\n\n  get model(): string {\n    return this.opts.model;\n  }\n\n  chat({\n    chatCtx,\n    toolCtx,\n    connOptions = DEFAULT_API_CONNECT_OPTIONS,\n    parallelToolCalls,\n    toolChoice,\n    // TODO(AJS-270): Add response_format parameter support\n    extraKwargs,\n  }: {\n    chatCtx: llm.ChatContext;\n    toolCtx?: llm.ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: llm.ToolChoice;\n    // TODO(AJS-270): Add responseFormat parameter\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream<TModel> {\n    let extras: Record<string, unknown> = { ...(extraKwargs || {}) };\n\n    if (this.opts.maxCompletionTokens !== undefined) {\n      extras.max_completion_tokens = this.opts.maxCompletionTokens;\n    }\n    if (this.opts.temperature !== undefined) {\n      extras.temperature = this.opts.temperature;\n    }\n    if (this.opts.verbosity !== undefined) {\n      extras.verbosity = this.opts.verbosity;\n    }\n\n    parallelToolCalls =\n      parallelToolCalls !== undefined ? parallelToolCalls : this.opts.parallelToolCalls;\n    if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) {\n      extras.parallel_tool_calls = parallelToolCalls;\n    }\n\n    toolChoice = toolChoice !== undefined ? toolChoice : this.opts.toolChoice;\n    if (toolChoice) {\n      extras.tool_choice = toolChoice;\n    }\n\n    // TODO(AJS-270): Add response_format support here\n\n    extras = { ...extras, ...this.opts.extraKwargs };\n\n    return new LLMStream(this, {\n      model: this.opts.model,\n      providerFmt: 'openai',\n      client: this.client,\n      chatCtx,\n      toolCtx,\n      connOptions,\n      extraKwargs: extras,\n      gatewayOptions: {\n        apiKey: this.opts.apiKey,\n        apiSecret: this.opts.apiSecret,\n      },\n    });\n  }\n}\n\nexport class LLMStream<TModel extends LLMModels> extends llm.LLMStream {\n  private model: TModel;\n  private providerFmt: llm.ProviderFormat;\n  private client: OpenAI;\n  private extraKwargs: Record<string, unknown>;\n\n  private gatewayOptions?: GatewayOptions;\n  private toolCallId?: string;\n  private toolIndex?: number;\n  private fncName?: string;\n  private fncRawArguments?: string;\n\n  constructor(\n    llm: LLM<TModel>,\n    {\n      model,\n      providerFmt,\n      client,\n      chatCtx,\n      toolCtx,\n      gatewayOptions,\n      connOptions,\n      extraKwargs,\n    }: {\n      model: TModel;\n      providerFmt: llm.ProviderFormat;\n      client: OpenAI;\n      chatCtx: llm.ChatContext;\n      toolCtx?: llm.ToolContext;\n      gatewayOptions?: GatewayOptions;\n      connOptions: APIConnectOptions;\n      extraKwargs: Record<string, any>;\n    },\n  ) {\n    super(llm, { chatCtx, toolCtx, connOptions });\n    this.client = client;\n    this.gatewayOptions = gatewayOptions;\n    this.providerFmt = providerFmt;\n    this.extraKwargs = extraKwargs;\n    this.model = model;\n  }\n\n  protected async run(): Promise<void> {\n    // current function call that we're waiting for full completion (args are streamed)\n    // (defined inside the run method to make sure the state is reset for each run/attempt)\n    let retryable = true;\n    this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = undefined;\n\n    try {\n      const messages = (await this.chatCtx.toProviderFormat(\n        this.providerFmt,\n      )) as OpenAI.ChatCompletionMessageParam[];\n\n      const tools = this.toolCtx\n        ? Object.entries(this.toolCtx).map(([name, func]) => ({\n            type: 'function' as const,\n            function: {\n              name,\n              description: func.description,\n              parameters: llm.toJsonSchema(\n                func.parameters,\n              ) as unknown as OpenAI.Chat.Completions.ChatCompletionTool['function']['parameters'],\n            },\n          }))\n        : undefined;\n\n      const requestExtras: Record<string, unknown> = { ...this.extraKwargs };\n      if (!tools) {\n        delete requestExtras.tool_choice;\n      }\n\n      // Dynamically set the access token for the LiveKit Agent Gateway API\n      if (this.gatewayOptions) {\n        this.client.apiKey = await createAccessToken(\n          this.gatewayOptions.apiKey,\n          this.gatewayOptions.apiSecret,\n        );\n      }\n\n      const stream = await this.client.chat.completions.create(\n        {\n          model: this.model,\n          messages,\n          tools,\n          stream: true,\n          stream_options: { include_usage: true },\n          ...requestExtras,\n        },\n        {\n          timeout: this.connOptions.timeoutMs,\n        },\n      );\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          if (this.abortController.signal.aborted) {\n            break;\n          }\n          const chatChunk = this.parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            retryable = false;\n            this.queue.put(chatChunk);\n          }\n        }\n\n        if (chunk.usage) {\n          const usage = chunk.usage;\n          retryable = false;\n          this.queue.put({\n            id: chunk.id,\n            usage: {\n              completionTokens: usage.completion_tokens,\n              promptTokens: usage.prompt_tokens,\n              promptCachedTokens: usage.prompt_tokens_details?.cached_tokens || 0,\n              totalTokens: usage.total_tokens,\n            },\n          });\n        }\n      }\n    } catch (error) {\n      if (error instanceof OpenAI.APIConnectionTimeoutError) {\n        throw new APITimeoutError({ options: { retryable } });\n      } else if (error instanceof OpenAI.APIError) {\n        throw new APIStatusError({\n          message: error.message,\n          options: {\n            statusCode: error.status,\n            body: error.error,\n            requestId: error.request_id,\n            retryable,\n          },\n        });\n      } else {\n        throw new APIConnectionError({\n          message: toError(error).message,\n          options: { retryable },\n        });\n      }\n    } finally {\n      this.queue.close();\n    }\n  }\n\n  private parseChoice(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    // https://github.com/livekit/agents/issues/688\n    // the delta can be None when using Azure OpenAI (content filtering)\n    if (delta === undefined) return undefined;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        /**\n         * The way OpenAI streams tool calls is a bit tricky.\n         *\n         * For any new tool call, it first emits a delta tool call with id, and function name,\n         * the rest of the delta chunks will only stream the remaining arguments string,\n         * until a new tool call is started or the tool call is finished.\n         * See below for an example.\n         *\n         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)\n         * [ChoiceDeltaToolCall(index=0, id='call_LaVeHWUHpef9K1sd5UO8TtLg', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]\n         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='{\"location\": \"P', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris}', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=1, id='call_ThU4OmMdQXnnVmpXGOCknXIB', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]\n         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='{\"location\": \"T', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='okyo', name=None), type=None)]\n         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=None), finish_reason='tool_calls', index=0, logprobs=None)\n         */\n        let callChunk: llm.ChatChunk | undefined;\n        // If we have a previous tool call and this is a new one, emit the previous\n        if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {\n          callChunk = this.createRunningToolCallChunk(id, delta);\n          this.toolCallId = this.fncName = this.fncRawArguments = undefined;\n        }\n\n        // Start or continue building the current tool call\n        if (tool.function.name) {\n          this.toolIndex = tool.index;\n          this.toolCallId = tool.id;\n          this.fncName = tool.function.name;\n          this.fncRawArguments = tool.function.arguments || '';\n        } else if (tool.function.arguments) {\n          this.fncRawArguments = (this.fncRawArguments || '') + tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    // If we're done with tool calls, emit the final one\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.toolCallId !== undefined\n    ) {\n      const callChunk = this.createRunningToolCallChunk(id, delta);\n      this.toolCallId = this.fncName = this.fncRawArguments = undefined;\n      return callChunk;\n    }\n\n    // Regular content message\n    if (!delta.content) {\n      return undefined;\n    }\n\n    return {\n      id,\n      delta: {\n        role: 'assistant',\n        content: delta.content,\n      },\n    };\n  }\n\n  private createRunningToolCallChunk(\n    id: string,\n    delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta,\n  ): llm.ChatChunk {\n    return {\n      id,\n      delta: {\n        role: 'assistant',\n        content: delta.content || undefined,\n        toolCalls: [\n          llm.FunctionCall.create({\n            callId: this.toolCallId || '',\n            name: this.fncName || '',\n            args: this.fncRawArguments || '',\n          }),\n        ],\n      },\n    };\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAmB;AACnB,eAMO;AACP,UAAqB;AAErB,mBAAkD;AAwElD,MAAM,mBAAmB;AAoBlB,MAAM,YAAsC,IAAI,IAAI;AAAA,EACjD;AAAA,EACA;AAAA,EAER,YAAY,MAaT;AACD,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,IAAI;AAEJ,UAAM,YAAY,WAAW,QAAQ,IAAI,yBAAyB;AAClE,UAAM,WAAW,UAAU,QAAQ,IAAI,6BAA6B,QAAQ,IAAI;AAChF,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAEA,UAAM,cACJ,aAAa,QAAQ,IAAI,gCAAgC,QAAQ,IAAI;AACvE,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI,MAAM,iEAAiE;AAAA,IACnF;AAEA,SAAK,OAAO;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,aAAa,eAAe,CAAC;AAAA,IAC/B;AAEA,SAAK,SAAS,IAAI,cAAAA,QAAO;AAAA,MACvB,SAAS,KAAK,KAAK;AAAA,MACnB,YAAY,cAAc;AAAA,MAC1B,SAAS,WAAW;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,EACF,GAQsB;AACpB,QAAI,SAAkC,EAAE,GAAI,eAAe,CAAC,EAAG;AAE/D,QAAI,KAAK,KAAK,wBAAwB,QAAW;AAC/C,aAAO,wBAAwB,KAAK,KAAK;AAAA,IAC3C;AACA,QAAI,KAAK,KAAK,gBAAgB,QAAW;AACvC,aAAO,cAAc,KAAK,KAAK;AAAA,IACjC;AACA,QAAI,KAAK,KAAK,cAAc,QAAW;AACrC,aAAO,YAAY,KAAK,KAAK;AAAA,IAC/B;AAEA,wBACE,sBAAsB,SAAY,oBAAoB,KAAK,KAAK;AAClE,QAAI,WAAW,OAAO,KAAK,OAAO,EAAE,SAAS,KAAK,sBAAsB,QAAW;AACjF,aAAO,sBAAsB;AAAA,IAC/B;AAEA,iBAAa,eAAe,SAAY,aAAa,KAAK,KAAK;AAC/D,QAAI,YAAY;AACd,aAAO,cAAc;AAAA,IACvB;AAIA,aAAS,EAAE,GAAG,QAAQ,GAAG,KAAK,KAAK,YAAY;AAE/C,WAAO,IAAI,UAAU,MAAM;AAAA,MACzB,OAAO,KAAK,KAAK;AAAA,MACjB,aAAa;AAAA,MACb,QAAQ,KAAK;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa;AAAA,MACb,gBAAgB;AAAA,QACd,QAAQ,KAAK,KAAK;AAAA,QAClB,WAAW,KAAK,KAAK;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,MAAM,kBAA4C,IAAI,UAAU;AAAA,EAC7D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YACEC,MACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAUA;AACA,UAAMA,MAAK,EAAE,SAAS,SAAS,YAAY,CAAC;AAC5C,SAAK,SAAS;AACd,SAAK,iBAAiB;AACtB,SAAK,cAAc;AACnB,SAAK,cAAc;AACnB,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,MAAgB,MAAqB;AA1RvC;AA6RI,QAAI,YAAY;AAChB,SAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB,KAAK,YAAY;AAEzE,QAAI;AACF,YAAM,WAAY,MAAM,KAAK,QAAQ;AAAA,QACnC,KAAK;AAAA,MACP;AAEA,YAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,OAAO;AAAA,QAClD,MAAM;AAAA,QACN,UAAU;AAAA,UACR;AAAA,UACA,aAAa,KAAK;AAAA,UAClB,YAAY,IAAI;AAAA,YACd,KAAK;AAAA,UACP;AAAA,QACF;AAAA,MACF,EAAE,IACF;AAEJ,YAAM,gBAAyC,EAAE,GAAG,KAAK,YAAY;AACrE,UAAI,CAAC,OAAO;AACV,eAAO,cAAc;AAAA,MACvB;AAGA,UAAI,KAAK,gBAAgB;AACvB,aAAK,OAAO,SAAS,UAAM;AAAA,UACzB,KAAK,eAAe;AAAA,UACpB,KAAK,eAAe;AAAA,QACtB;AAAA,MACF;AAEA,YAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,QAChD;AAAA,UACE,OAAO,KAAK;AAAA,UACZ;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,UACR,gBAAgB,EAAE,eAAe,KAAK;AAAA,UACtC,GAAG;AAAA,QACL;AAAA,QACA;AAAA,UACE,SAAS,KAAK,YAAY;AAAA,QAC5B;AAAA,MACF;AAEA,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,cAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,UACF;AACA,gBAAM,YAAY,KAAK,YAAY,MAAM,IAAI,MAAM;AACnD,cAAI,WAAW;AACb,wBAAY;AACZ,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAAA,QACF;AAEA,YAAI,MAAM,OAAO;AACf,gBAAM,QAAQ,MAAM;AACpB,sBAAY;AACZ,eAAK,MAAM,IAAI;AAAA,YACb,IAAI,MAAM;AAAA,YACV,OAAO;AAAA,cACL,kBAAkB,MAAM;AAAA,cACxB,cAAc,MAAM;AAAA,cACpB,sBAAoB,WAAM,0BAAN,mBAA6B,kBAAiB;AAAA,cAClE,aAAa,MAAM;AAAA,YACrB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAD,QAAO,2BAA2B;AACrD,cAAM,IAAI,yBAAgB,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;AAAA,MACtD,WAAW,iBAAiB,cAAAA,QAAO,UAAU;AAC3C,cAAM,IAAI,wBAAe;AAAA,UACvB,SAAS,MAAM;AAAA,UACf,SAAS;AAAA,YACP,YAAY,MAAM;AAAA,YAClB,MAAM,MAAM;AAAA,YACZ,WAAW,MAAM;AAAA,YACjB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH,OAAO;AACL,cAAM,IAAI,4BAAmB;AAAA,UAC3B,aAAS,kBAAQ,KAAK,EAAE;AAAA,UACxB,SAAS,EAAE,UAAU;AAAA,QACvB,CAAC;AAAA,MACH;AAAA,IACF,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AAAA,EAEQ,YACN,IACA,QAC2B;AAC3B,UAAM,QAAQ,OAAO;AAIrB,QAAI,UAAU,OAAW,QAAO;AAEhC,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAmBA,YAAI;AAEJ,YAAI,KAAK,cAAc,KAAK,MAAM,KAAK,UAAU,KAAK,WAAW;AAC/D,sBAAY,KAAK,2BAA2B,IAAI,KAAK;AACrD,eAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB;AAAA,QAC1D;AAGA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,YAAY,KAAK;AACtB,eAAK,aAAa,KAAK;AACvB,eAAK,UAAU,KAAK,SAAS;AAC7B,eAAK,kBAAkB,KAAK,SAAS,aAAa;AAAA,QACpD,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,mBAAmB,KAAK,mBAAmB,MAAM,KAAK,SAAS;AAAA,QACtE;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAGA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,eAAe,QACpB;AACA,YAAM,YAAY,KAAK,2BAA2B,IAAI,KAAK;AAC3D,WAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB;AACxD,aAAO;AAAA,IACT;AAGA,QAAI,CAAC,MAAM,SAAS;AAClB,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,MAAM;AAAA,QACN,SAAS,MAAM;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,2BACN,IACA,OACe;AACf,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,MAAM;AAAA,QACN,SAAS,MAAM,WAAW;AAAA,QAC1B,WAAW;AAAA,UACT,IAAI,aAAa,OAAO;AAAA,YACtB,QAAQ,KAAK,cAAc;AAAA,YAC3B,MAAM,KAAK,WAAW;AAAA,YACtB,MAAM,KAAK,mBAAmB;AAAA,UAChC,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":["OpenAI","llm"]}

package/dist/inference/llm.d.cts

@@ -0,0 +1,92 @@
+import OpenAI from 'openai';
+import * as llm from '../llm/index.js';
+import type { APIConnectOptions } from '../types.js';
+import { type AnyModels } from './utils.js';
+export type AzureModels = 'azure/gpt-4.1' | 'azure/gpt-4.1-mini' | 'azure/gpt-4.1-nano';
+export type CerebrasModels = 'cerebras/llama3.1-8b' | 'cerebras/llama-3.3-70b' | 'cerebras/llama-4-scout-17b-16e-instruct' | 'cerebras/gpt-oss-120b' | 'cerebras/qwen-3-32b' | 'cerebras/llama-4-maverick-17b-128e-instruct' | 'cerebras/qwen-3-235b-a22b-instruct-2507';
+export type GroqModels = 'groq/llama-3.1-8b-instant' | 'groq/llama-3.3-70b-versatile' | 'groq/openai/gpt-oss-120b' | 'groq/openai/gpt-oss-20b' | 'groq/meta-llama/llama-4-maverick-17b-128e-instruct' | 'groq/meta-llama/llama-4-scout-17b-16e-instruct' | 'groq/qwen/qwen3-32b';
+export type BasetenModels = 'baseten/deepseek-ai/DeepSeek-V3-0324' | 'baseten/meta-llama/Llama-4-Scout-17B-16E-Instruct' | 'baseten/meta-llama/Llama-4-Maverick-17B-128E-Instruct' | 'baseten/moonshotai/Kimi-K2-Instruct' | 'baseten/openai/gpt-oss-120b' | 'baseten/Qwen/Qwen3-235B-A22B-Instruct-2507';
+export interface AzureOptions {
+    top_p?: number;
+}
+export interface CerebrasOptions {
+    top_p?: number;
+}
+export interface GroqOptions {
+    top_p?: number;
+}
+export interface BasetenOptions {
+    top_p?: number;
+}
+export type LLMModels = AzureModels | CerebrasModels | GroqModels | BasetenModels | AnyModels;
+export type LLMOptions<T extends LLMModels> = T extends AzureModels ? AzureOptions : T extends CerebrasModels ? CerebrasOptions : T extends GroqOptions ? GroqOptions : T extends BasetenOptions ? BasetenOptions : Record<string, unknown>;
+export type Verbosity = 'low' | 'medium' | 'high';
+export interface InferenceLLMOptions<TModel extends LLMModels> {
+    model: TModel;
+    temperature?: number;
+    parallelToolCalls?: boolean;
+    toolChoice?: llm.ToolChoice;
+    maxCompletionTokens?: number;
+    baseURL: string;
+    apiKey: string;
+    apiSecret: string;
+    verbosity?: Verbosity;
+    extraKwargs: Record<string, any>;
+}
+export interface GatewayOptions {
+    apiKey: string;
+    apiSecret: string;
+}
+export declare class LLM<TModel extends LLMModels> extends llm.LLM {
+    private client;
+    private opts;
+    constructor(opts: {
+        model: TModel;
+        temperature?: number;
+        parallelToolCalls?: boolean;
+        toolChoice?: llm.ToolChoice;
+        maxCompletionTokens?: number;
+        baseURL?: string;
+        apiKey?: string;
+        apiSecret?: string;
+        maxRetries?: number;
+        timeout?: number;
+        verbosity?: Verbosity;
+        extraKwargs?: LLMOptions<TModel>;
+    });
+    label(): string;
+    get model(): string;
+    chat({ chatCtx, toolCtx, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
+        chatCtx: llm.ChatContext;
+        toolCtx?: llm.ToolContext;
+        connOptions?: APIConnectOptions;
+        parallelToolCalls?: boolean;
+        toolChoice?: llm.ToolChoice;
+        extraKwargs?: Record<string, unknown>;
+    }): LLMStream<TModel>;
+}
+export declare class LLMStream<TModel extends LLMModels> extends llm.LLMStream {
+    private model;
+    private providerFmt;
+    private client;
+    private extraKwargs;
+    private gatewayOptions?;
+    private toolCallId?;
+    private toolIndex?;
+    private fncName?;
+    private fncRawArguments?;
+    constructor(llm: LLM<TModel>, { model, providerFmt, client, chatCtx, toolCtx, gatewayOptions, connOptions, extraKwargs, }: {
+        model: TModel;
+        providerFmt: llm.ProviderFormat;
+        client: OpenAI;
+        chatCtx: llm.ChatContext;
+        toolCtx?: llm.ToolContext;
+        gatewayOptions?: GatewayOptions;
+        connOptions: APIConnectOptions;
+        extraKwargs: Record<string, any>;
+    });
+    protected run(): Promise<void>;
+    private parseChoice;
+    private createRunningToolCallChunk;
+}
+//# sourceMappingURL=llm.d.ts.map