@livekit/agents 0.7.8 → 1.0.0-next.0

package/dist/llm/llm.cjs

@@ -19,31 +19,105 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var llm_exports = {};
 __export(llm_exports, {
   LLM: () => LLM,
-  LLMEvent: () => LLMEvent,
   LLMStream: () => LLMStream
 });
 module.exports = __toCommonJS(llm_exports);
+var import_async = require("@std/async");
 var import_node_events = require("node:events");
+var import_exceptions = require("../_exceptions.cjs");
+var import_log = require("../log.cjs");
 var import_utils = require("../utils.cjs");
-var LLMEvent = /* @__PURE__ */ ((LLMEvent2) => {
-  LLMEvent2[LLMEvent2["METRICS_COLLECTED"] = 0] = "METRICS_COLLECTED";
-  return LLMEvent2;
-})(LLMEvent || {});
+var import_chat_context = require("./chat_context.cjs");
 class LLM extends import_node_events.EventEmitter {
+  constructor() {
+    super();
+  }
+  /**
+   * Get the model name/identifier for this LLM instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model() {
+    return "unknown";
+  }
+  /**
+   * Pre-warm connection to the LLM service
+   */
+  prewarm() {
+  }
+  async aclose() {
+  }
 }
 class LLMStream {
   output = new import_utils.AsyncIterableQueue();
   queue = new import_utils.AsyncIterableQueue();
   closed = false;
-  _functionCalls = [];
+  abortController = new AbortController();
+  _connOptions;
+  logger = (0, import_log.log)();
   #llm;
   #chatCtx;
-  #fncCtx;
-  constructor(llm, chatCtx, fncCtx) {
+  #toolCtx;
+  constructor(llm, {
+    chatCtx,
+    toolCtx,
+    connOptions
+  }) {
     this.#llm = llm;
     this.#chatCtx = chatCtx;
-    this.#fncCtx = fncCtx;
+    this.#toolCtx = toolCtx;
+    this._connOptions = connOptions;
     this.monitorMetrics();
+    this.abortController.signal.addEventListener("abort", () => {
+      this.output.close();
+      this.closed = true;
+    });
+    (0, import_utils.startSoon)(() => this.mainTask().then(() => this.queue.close()));
+  }
+  async mainTask() {
+    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+      try {
+        return await this.run();
+      } catch (error) {
+        if (error instanceof import_exceptions.APIError) {
+          const retryInterval = this._connOptions._intervalForRetry(i);
+          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+            this.emitError({ error, recoverable: false });
+            throw error;
+          } else if (i === this._connOptions.maxRetry) {
+            this.emitError({ error, recoverable: false });
+            throw new import_exceptions.APIConnectionError({
+              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,
+              options: { retryable: false }
+            });
+          } else {
+            this.emitError({ error, recoverable: true });
+            this.logger.warn(
+              { llm: this.#llm.label(), attempt: i + 1, error },
+              `failed to generate LLM completion, retrying in ${retryInterval}s`
+            );
+          }
+          if (retryInterval > 0) {
+            await (0, import_async.delay)(retryInterval);
+          }
+        } else {
+          this.emitError({ error: (0, import_utils.toError)(error), recoverable: false });
+          throw error;
+        }
+      }
+    }
+  }
+  emitError({ error, recoverable }) {
+    this.#llm.emit("error", {
+      type: "llm_error",
+      timestamp: Date.now(),
+      label: this.#llm.label(),
+      error,
+      recoverable
+    });
   }
   async monitorMetrics() {
     const startTime = process.hrtime.bigint();
@@ -51,8 +125,11 @@ class LLMStream {
     let requestId = "";
     let usage;
     for await (const ev of this.queue) {
+      if (this.abortController.signal.aborted) {
+        break;
+      }
       this.output.put(ev);
-      requestId = ev.requestId;
+      requestId = ev.id;
       if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
       }
@@ -63,49 +140,38 @@ class LLMStream {
     this.output.close();
     const duration = process.hrtime.bigint() - startTime;
     const metrics = {
+      type: "llm_metrics",
       timestamp: Date.now(),
       requestId,
       ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1e6))),
       duration: Math.trunc(Number(duration / BigInt(1e6))),
-      cancelled: false,
-      // XXX(nbsp)
-      label: this.label,
+      cancelled: this.abortController.signal.aborted,
+      label: this.#llm.label(),
       completionTokens: (usage == null ? void 0 : usage.completionTokens) || 0,
       promptTokens: (usage == null ? void 0 : usage.promptTokens) || 0,
+      promptCachedTokens: (usage == null ? void 0 : usage.promptCachedTokens) || 0,
       totalTokens: (usage == null ? void 0 : usage.totalTokens) || 0,
       tokensPerSecond: ((usage == null ? void 0 : usage.completionTokens) || 0) / Math.trunc(Number(duration / BigInt(1e9)))
     };
-    this.#llm.emit(0 /* METRICS_COLLECTED */, metrics);
-  }
-  /** List of called functions from this stream. */
-  get functionCalls() {
-    return this._functionCalls;
+    this.#llm.emit("metrics_collected", metrics);
   }
   /** The function context of this stream. */
-  get fncCtx() {
-    return this.#fncCtx;
+  get toolCtx() {
+    return this.#toolCtx;
   }
   /** The initial chat context of this stream. */
   get chatCtx() {
     return this.#chatCtx;
   }
-  /** Execute all deferred functions of this stream concurrently. */
-  executeFunctions() {
-    this._functionCalls.forEach(
-      (f) => f.task = f.func.execute(f.params).then(
-        (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),
-        (error) => ({ name: f.name, toolCallId: f.toolCallId, error })
-      )
-    );
-    return this._functionCalls;
+  /** The connection options for this stream. */
+  get connOptions() {
+    return this._connOptions;
   }
   next() {
     return this.output.next();
   }
   close() {
-    this.output.close();
-    this.queue.close();
-    this.closed = true;
+    this.abortController.abort();
   }
   [Symbol.asyncIterator]() {
     return this;
@@ -114,7 +180,6 @@ class LLMStream {
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   LLM,
-  LLMEvent,
   LLMStream
 });
 //# sourceMappingURL=llm.cjs.map

package/dist/llm/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { ChatContext, ChatRole } from './chat_context.js';\nimport type { FunctionCallInfo, FunctionContext } from './function_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCallInfo[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n}\n\nexport interface Choice {\n  delta: ChoiceDelta;\n  index: number;\n}\n\nexport interface ChatChunk {\n  requestId: string;\n  choices: Choice[];\n  usage?: CompletionUsage;\n}\n\nexport enum LLMEvent {\n  METRICS_COLLECTED,\n}\n\nexport type LLMCallbacks = {\n  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: ChatContext;\n    fncCtx?: FunctionContext;\n    temperature?: number;\n    n?: number;\n    parallelToolCalls?: boolean;\n  }): LLMStream;\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected _functionCalls: FunctionCallInfo[] = [];\n  abstract label: string;\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #fncCtx?: FunctionContext;\n\n  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      this.output.put(ev);\n      requestId = ev.requestId;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** List of called functions from this stream. */\n  get functionCalls(): FunctionCallInfo[] {\n    return this._functionCalls;\n  }\n\n  /** The function context of this stream. */\n  get fncCtx(): FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** Execute all deferred functions of this stream concurrently. */\n  executeFunctions(): FunctionCallInfo[] {\n    this._functionCalls.forEach(\n      (f) =>\n        (f.task = f.func.execute(f.params).then(\n          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),\n          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),\n        )),\n    );\n    return this._functionCalls;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.output.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAA6B;AAE7B,mBAAmC;AA2B5B,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAQL,MAAe,YAAa,gCAAsD;AAiBzF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,iBAAqC,CAAC;AAAA,EAGhD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,SAAsB,QAA0B;AACpE,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,IAAI,gBAAoC;AACtC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,mBAAuC;AACrC,SAAK,eAAe;AAAA,MAClB,CAAC,MACE,EAAE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE;AAAA,QACjC,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,OAAO;AAAA,QAC9D,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,MAAM;AAAA,MAC9D;AAAA,IACJ;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,MAAM;AAClB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["LLMEvent"]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { delay } from '@std/async';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { AsyncIterableQueue, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, any>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAsB;AACtB,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAGpB,mBAAuD;AACvD,0BAAmE;AAmC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/llm/llm.d.cts

@@ -1,63 +1,89 @@
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { LLMMetrics } from '../metrics/base.js';
+import type { APIConnectOptions } from '../types.js';
 import { AsyncIterableQueue } from '../utils.js';
-import type { ChatContext, ChatRole } from './chat_context.js';
-import type { FunctionCallInfo, FunctionContext } from './function_context.js';
+import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
 export interface ChoiceDelta {
     role: ChatRole;
     content?: string;
-    toolCalls?: FunctionCallInfo[];
+    toolCalls?: FunctionCall[];
 }
 export interface CompletionUsage {
     completionTokens: number;
     promptTokens: number;
+    promptCachedTokens: number;
     totalTokens: number;
 }
-export interface Choice {
-    delta: ChoiceDelta;
-    index: number;
-}
 export interface ChatChunk {
-    requestId: string;
-    choices: Choice[];
+    id: string;
+    delta?: ChoiceDelta;
     usage?: CompletionUsage;
 }
-export declare enum LLMEvent {
-    METRICS_COLLECTED = 0
+export interface LLMError {
+    type: 'llm_error';
+    timestamp: number;
+    label: string;
+    error: Error;
+    recoverable: boolean;
 }
 export type LLMCallbacks = {
-    [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+    ['metrics_collected']: (metrics: LLMMetrics) => void;
+    ['error']: (error: LLMError) => void;
 };
 declare const LLM_base: new () => TypedEmitter<LLMCallbacks>;
 export declare abstract class LLM extends LLM_base {
+    constructor();
+    abstract label(): string;
+    /**
+     * Get the model name/identifier for this LLM instance.
+     *
+     * @returns The model name if available, "unknown" otherwise.
+     *
+     * @remarks
+     * Plugins should override this property to provide their model information.
+     */
+    get model(): string;
     /**
      * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
      */
-    abstract chat({ chatCtx, fncCtx, temperature, n, parallelToolCalls, }: {
+    abstract chat({ chatCtx, toolCtx, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
         chatCtx: ChatContext;
-        fncCtx?: FunctionContext;
-        temperature?: number;
-        n?: number;
+        toolCtx?: ToolContext;
+        connOptions?: APIConnectOptions;
         parallelToolCalls?: boolean;
+        toolChoice?: ToolChoice;
+        extraKwargs?: Record<string, any>;
     }): LLMStream;
+    /**
+     * Pre-warm connection to the LLM service
+     */
+    prewarm(): void;
+    aclose(): Promise<void>;
 }
 export declare abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     #private;
     protected output: AsyncIterableQueue<ChatChunk>;
     protected queue: AsyncIterableQueue<ChatChunk>;
     protected closed: boolean;
-    protected _functionCalls: FunctionCallInfo[];
-    abstract label: string;
-    constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext);
+    protected abortController: AbortController;
+    protected _connOptions: APIConnectOptions;
+    protected logger: import("pino").Logger;
+    constructor(llm: LLM, { chatCtx, toolCtx, connOptions, }: {
+        chatCtx: ChatContext;
+        toolCtx?: ToolContext;
+        connOptions: APIConnectOptions;
+    });
+    private mainTask;
+    private emitError;
     protected monitorMetrics(): Promise<void>;
-    /** List of called functions from this stream. */
-    get functionCalls(): FunctionCallInfo[];
+    protected abstract run(): Promise<void>;
     /** The function context of this stream. */
-    get fncCtx(): FunctionContext | undefined;
+    get toolCtx(): ToolContext | undefined;
     /** The initial chat context of this stream. */
     get chatCtx(): ChatContext;
-    /** Execute all deferred functions of this stream concurrently. */
-    executeFunctions(): FunctionCallInfo[];
+    /** The connection options for this stream. */
+    get connOptions(): APIConnectOptions;
     next(): Promise<IteratorResult<ChatChunk>>;
     close(): void;
     [Symbol.asyncIterator](): LLMStream;

package/dist/llm/llm.d.ts

@@ -1,63 +1,89 @@
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { LLMMetrics } from '../metrics/base.js';
+import type { APIConnectOptions } from '../types.js';
 import { AsyncIterableQueue } from '../utils.js';
-import type { ChatContext, ChatRole } from './chat_context.js';
-import type { FunctionCallInfo, FunctionContext } from './function_context.js';
+import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
 export interface ChoiceDelta {
     role: ChatRole;
     content?: string;
-    toolCalls?: FunctionCallInfo[];
+    toolCalls?: FunctionCall[];
 }
 export interface CompletionUsage {
     completionTokens: number;
     promptTokens: number;
+    promptCachedTokens: number;
     totalTokens: number;
 }
-export interface Choice {
-    delta: ChoiceDelta;
-    index: number;
-}
 export interface ChatChunk {
-    requestId: string;
-    choices: Choice[];
+    id: string;
+    delta?: ChoiceDelta;
     usage?: CompletionUsage;
 }
-export declare enum LLMEvent {
-    METRICS_COLLECTED = 0
+export interface LLMError {
+    type: 'llm_error';
+    timestamp: number;
+    label: string;
+    error: Error;
+    recoverable: boolean;
 }
 export type LLMCallbacks = {
-    [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+    ['metrics_collected']: (metrics: LLMMetrics) => void;
+    ['error']: (error: LLMError) => void;
 };
 declare const LLM_base: new () => TypedEmitter<LLMCallbacks>;
 export declare abstract class LLM extends LLM_base {
+    constructor();
+    abstract label(): string;
+    /**
+     * Get the model name/identifier for this LLM instance.
+     *
+     * @returns The model name if available, "unknown" otherwise.
+     *
+     * @remarks
+     * Plugins should override this property to provide their model information.
+     */
+    get model(): string;
     /**
      * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
      */
-    abstract chat({ chatCtx, fncCtx, temperature, n, parallelToolCalls, }: {
+    abstract chat({ chatCtx, toolCtx, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
         chatCtx: ChatContext;
-        fncCtx?: FunctionContext;
-        temperature?: number;
-        n?: number;
+        toolCtx?: ToolContext;
+        connOptions?: APIConnectOptions;
         parallelToolCalls?: boolean;
+        toolChoice?: ToolChoice;
+        extraKwargs?: Record<string, any>;
     }): LLMStream;
+    /**
+     * Pre-warm connection to the LLM service
+     */
+    prewarm(): void;
+    aclose(): Promise<void>;
 }
 export declare abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     #private;
     protected output: AsyncIterableQueue<ChatChunk>;
     protected queue: AsyncIterableQueue<ChatChunk>;
     protected closed: boolean;
-    protected _functionCalls: FunctionCallInfo[];
-    abstract label: string;
-    constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext);
+    protected abortController: AbortController;
+    protected _connOptions: APIConnectOptions;
+    protected logger: import("pino").Logger;
+    constructor(llm: LLM, { chatCtx, toolCtx, connOptions, }: {
+        chatCtx: ChatContext;
+        toolCtx?: ToolContext;
+        connOptions: APIConnectOptions;
+    });
+    private mainTask;
+    private emitError;
     protected monitorMetrics(): Promise<void>;
-    /** List of called functions from this stream. */
-    get functionCalls(): FunctionCallInfo[];
+    protected abstract run(): Promise<void>;
     /** The function context of this stream. */
-    get fncCtx(): FunctionContext | undefined;
+    get toolCtx(): ToolContext | undefined;
     /** The initial chat context of this stream. */
     get chatCtx(): ChatContext;
-    /** Execute all deferred functions of this stream concurrently. */
-    executeFunctions(): FunctionCallInfo[];
+    /** The connection options for this stream. */
+    get connOptions(): APIConnectOptions;
     next(): Promise<IteratorResult<ChatChunk>>;
     close(): void;
     [Symbol.asyncIterator](): LLMStream;

package/dist/llm/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE/E,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,oBAAY,QAAQ;IAClB,iBAAiB,IAAA;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;CAC7D,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;IACtF;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,MAAM,EACN,WAAW,EACX,CAAC,EACD,iBAAiB,GAClB,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,iBAAiB,CAAC,EAAE,OAAO,CAAC;KAC7B,GAAG,SAAS;CACd;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,MAAM,gCAAuC;IACvD,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,cAAc,EAAE,gBAAgB,EAAE,CAAM;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAMX,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,eAAe;cAOpD,cAAc;IAmC9B,iDAAiD;IACjD,IAAI,aAAa,IAAI,gBAAgB,EAAE,CAEtC;IAED,2CAA2C;IAC3C,IAAI,MAAM,IAAI,eAAe,GAAG,SAAS,CAExC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,kEAAkE;IAClE,gBAAgB,IAAI,gBAAgB,EAAE;IAWtC,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAML,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAKhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAsB,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACvF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAKtF,QAAQ,CAAC,KAAK,IAAI,MAAM;IAExB;;;;;;;OAOG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,UAAU,EACV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KACnC,GAAG,SAAS;IAEb;;OAEG;IACH,OAAO,IAAI,IAAI;IAIT,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,MAAM,gCAAuC;IACvD,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,eAAe,kBAAyB;IAClD,SAAS,CAAC,YAAY,EAAE,iBAAiB,CAAC;IAC1C,SAAS,CAAC,MAAM,wBAAS;gBAOvB,GAAG,EAAE,GAAG,EACR,EACE,OAAO,EACP,OAAO,EACP,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,EAAE,iBAAiB,CAAC;KAChC;YAoBW,QAAQ;IAoCtB,OAAO,CAAC,SAAS;cAUD,cAAc;IAwC9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,2CAA2C;IAC3C,IAAI,OAAO,IAAI,WAAW,GAAG,SAAS,CAErC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,8CAA8C;IAC9C,IAAI,WAAW,IAAI,iBAAiB,CAEnC;IAED,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}

package/dist/llm/llm.js

@@ -1,24 +1,99 @@
+import { delay } from "@std/async";
 import { EventEmitter } from "node:events";
-import { AsyncIterableQueue } from "../utils.js";
-var LLMEvent = /* @__PURE__ */ ((LLMEvent2) => {
-  LLMEvent2[LLMEvent2["METRICS_COLLECTED"] = 0] = "METRICS_COLLECTED";
-  return LLMEvent2;
-})(LLMEvent || {});
+import { APIConnectionError, APIError } from "../_exceptions.js";
+import { log } from "../log.js";
+import { AsyncIterableQueue, startSoon, toError } from "../utils.js";
+import {} from "./chat_context.js";
 class LLM extends EventEmitter {
+  constructor() {
+    super();
+  }
+  /**
+   * Get the model name/identifier for this LLM instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model() {
+    return "unknown";
+  }
+  /**
+   * Pre-warm connection to the LLM service
+   */
+  prewarm() {
+  }
+  async aclose() {
+  }
 }
 class LLMStream {
   output = new AsyncIterableQueue();
   queue = new AsyncIterableQueue();
   closed = false;
-  _functionCalls = [];
+  abortController = new AbortController();
+  _connOptions;
+  logger = log();
   #llm;
   #chatCtx;
-  #fncCtx;
-  constructor(llm, chatCtx, fncCtx) {
+  #toolCtx;
+  constructor(llm, {
+    chatCtx,
+    toolCtx,
+    connOptions
+  }) {
     this.#llm = llm;
     this.#chatCtx = chatCtx;
-    this.#fncCtx = fncCtx;
+    this.#toolCtx = toolCtx;
+    this._connOptions = connOptions;
     this.monitorMetrics();
+    this.abortController.signal.addEventListener("abort", () => {
+      this.output.close();
+      this.closed = true;
+    });
+    startSoon(() => this.mainTask().then(() => this.queue.close()));
+  }
+  async mainTask() {
+    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+      try {
+        return await this.run();
+      } catch (error) {
+        if (error instanceof APIError) {
+          const retryInterval = this._connOptions._intervalForRetry(i);
+          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+            this.emitError({ error, recoverable: false });
+            throw error;
+          } else if (i === this._connOptions.maxRetry) {
+            this.emitError({ error, recoverable: false });
+            throw new APIConnectionError({
+              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,
+              options: { retryable: false }
+            });
+          } else {
+            this.emitError({ error, recoverable: true });
+            this.logger.warn(
+              { llm: this.#llm.label(), attempt: i + 1, error },
+              `failed to generate LLM completion, retrying in ${retryInterval}s`
+            );
+          }
+          if (retryInterval > 0) {
+            await delay(retryInterval);
+          }
+        } else {
+          this.emitError({ error: toError(error), recoverable: false });
+          throw error;
+        }
+      }
+    }
+  }
+  emitError({ error, recoverable }) {
+    this.#llm.emit("error", {
+      type: "llm_error",
+      timestamp: Date.now(),
+      label: this.#llm.label(),
+      error,
+      recoverable
+    });
   }
   async monitorMetrics() {
     const startTime = process.hrtime.bigint();
@@ -26,8 +101,11 @@ class LLMStream {
     let requestId = "";
     let usage;
     for await (const ev of this.queue) {
+      if (this.abortController.signal.aborted) {
+        break;
+      }
       this.output.put(ev);
-      requestId = ev.requestId;
+      requestId = ev.id;
       if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
       }
@@ -38,49 +116,38 @@ class LLMStream {
     this.output.close();
     const duration = process.hrtime.bigint() - startTime;
     const metrics = {
+      type: "llm_metrics",
       timestamp: Date.now(),
       requestId,
       ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1e6))),
       duration: Math.trunc(Number(duration / BigInt(1e6))),
-      cancelled: false,
-      // XXX(nbsp)
-      label: this.label,
+      cancelled: this.abortController.signal.aborted,
+      label: this.#llm.label(),
       completionTokens: (usage == null ? void 0 : usage.completionTokens) || 0,
       promptTokens: (usage == null ? void 0 : usage.promptTokens) || 0,
+      promptCachedTokens: (usage == null ? void 0 : usage.promptCachedTokens) || 0,
       totalTokens: (usage == null ? void 0 : usage.totalTokens) || 0,
       tokensPerSecond: ((usage == null ? void 0 : usage.completionTokens) || 0) / Math.trunc(Number(duration / BigInt(1e9)))
     };
-    this.#llm.emit(0 /* METRICS_COLLECTED */, metrics);
-  }
-  /** List of called functions from this stream. */
-  get functionCalls() {
-    return this._functionCalls;
+    this.#llm.emit("metrics_collected", metrics);
   }
   /** The function context of this stream. */
-  get fncCtx() {
-    return this.#fncCtx;
+  get toolCtx() {
+    return this.#toolCtx;
   }
   /** The initial chat context of this stream. */
   get chatCtx() {
     return this.#chatCtx;
   }
-  /** Execute all deferred functions of this stream concurrently. */
-  executeFunctions() {
-    this._functionCalls.forEach(
-      (f) => f.task = f.func.execute(f.params).then(
-        (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),
-        (error) => ({ name: f.name, toolCallId: f.toolCallId, error })
-      )
-    );
-    return this._functionCalls;
+  /** The connection options for this stream. */
+  get connOptions() {
+    return this._connOptions;
   }
   next() {
     return this.output.next();
   }
   close() {
-    this.output.close();
-    this.queue.close();
-    this.closed = true;
+    this.abortController.abort();
   }
   [Symbol.asyncIterator]() {
     return this;
@@ -88,7 +155,6 @@ class LLMStream {
 }
 export {
   LLM,
-  LLMEvent,
   LLMStream
 };
 //# sourceMappingURL=llm.js.map