@livekit/agents 1.1.0 → 1.2.1

package/dist/llm/fallback_adapter.test.js

@@ -0,0 +1,175 @@
+import { beforeAll, describe, expect, it, vi } from "vitest";
+import { APIConnectionError, APIError } from "../_exceptions.js";
+import { initializeLogger } from "../log.js";
+import { delay } from "../utils.js";
+import { FallbackAdapter } from "./fallback_adapter.js";
+import { LLM, LLMStream } from "./llm.js";
+class MockLLMStream extends LLMStream {
+  constructor(llm, opts, shouldFail = false, failAfterChunks = 0) {
+    super(llm, opts);
+    this.shouldFail = shouldFail;
+    this.failAfterChunks = failAfterChunks;
+    this.myLLM = llm;
+  }
+  myLLM;
+  async run() {
+    if (this.shouldFail && this.failAfterChunks === 0) {
+      throw new APIError("Mock LLM failed immediately");
+    }
+    const chunk = {
+      id: "test-id",
+      delta: { role: "assistant", content: "chunk" }
+    };
+    for (let i = 0; i < 3; i++) {
+      if (this.shouldFail && i === this.failAfterChunks) {
+        throw new APIError("Mock LLM failed after chunks");
+      }
+      this.queue.put(chunk);
+      await delay(10);
+    }
+  }
+}
+class MockLLM extends LLM {
+  shouldFail = false;
+  failAfterChunks = 0;
+  _label;
+  constructor(label) {
+    super();
+    this._label = label;
+  }
+  label() {
+    return this._label;
+  }
+  chat(opts) {
+    return new MockLLMStream(
+      this,
+      {
+        chatCtx: opts.chatCtx,
+        toolCtx: opts.toolCtx,
+        connOptions: opts.connOptions
+      },
+      this.shouldFail,
+      this.failAfterChunks
+    );
+  }
+}
+describe("FallbackAdapter", () => {
+  beforeAll(() => {
+    initializeLogger({ pretty: false });
+    process.on("unhandledRejection", () => {
+    });
+  });
+  it("should initialize correctly", () => {
+    const llm1 = new MockLLM("llm1");
+    const adapter = new FallbackAdapter({ llms: [llm1] });
+    expect(adapter.llms).toHaveLength(1);
+    expect(adapter.llms[0]).toBe(llm1);
+  });
+  it("should throw if no LLMs provided", () => {
+    expect(() => new FallbackAdapter({ llms: [] })).toThrow();
+  });
+  it("should use primary LLM if successful", async () => {
+    const llm1 = new MockLLM("llm1");
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    expect(chunks).toHaveLength(3);
+  });
+  it("should fallback to second LLM if first fails immediately", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    expect(chunks).toHaveLength(3);
+    expect(adapter._status[0].available).toBe(false);
+    expect(adapter._status[1].available).toBe(true);
+  });
+  it("should fail if all LLMs fail", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM("llm2");
+    llm2.shouldFail = true;
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const errorPromise = new Promise((resolve) => {
+      adapter.on("error", (e) => resolve(e.error));
+    });
+    for await (const _ of stream) {
+    }
+    const error = await errorPromise;
+    expect(error).toBeInstanceOf(APIConnectionError);
+  });
+  it("should fail if chunks sent and retryOnChunkSent is false", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    llm1.failAfterChunks = 1;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({
+      llms: [llm1, llm2],
+      retryOnChunkSent: false
+    });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const errorPromise = new Promise((resolve) => {
+      adapter.on("error", (e) => resolve(e.error));
+    });
+    for await (const _ of stream) {
+    }
+    const error = await errorPromise;
+    expect(error).toBeInstanceOf(APIError);
+  });
+  it("should fallback if chunks sent and retryOnChunkSent is true", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    llm1.failAfterChunks = 1;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({
+      llms: [llm1, llm2],
+      retryOnChunkSent: true
+    });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    expect(chunks).toHaveLength(4);
+  });
+  it("should emit availability changed events", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const eventSpy = vi.fn();
+    adapter.on("llm_availability_changed", eventSpy);
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    for await (const _ of stream) {
+    }
+    expect(eventSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        llm: llm1,
+        available: false
+      })
+    );
+  });
+});
+//# sourceMappingURL=fallback_adapter.test.js.map

package/dist/llm/fallback_adapter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/llm/fallback_adapter.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { beforeAll, describe, expect, it, vi } from 'vitest';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { initializeLogger } from '../log.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { delay } from '../utils.js';\nimport type { ChatContext } from './chat_context.js';\nimport { FallbackAdapter } from './fallback_adapter.js';\nimport { type ChatChunk, LLM, LLMStream } from './llm.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nclass MockLLMStream extends LLMStream {\n  public myLLM: LLM;\n\n  constructor(\n    llm: LLM,\n    opts: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n    private shouldFail: boolean = false,\n    private failAfterChunks: number = 0,\n  ) {\n    super(llm, opts);\n    this.myLLM = llm;\n  }\n\n  protected async run(): Promise<void> {\n    if (this.shouldFail && this.failAfterChunks === 0) {\n      throw new APIError('Mock LLM failed immediately');\n    }\n\n    const chunk: ChatChunk = {\n      id: 'test-id',\n      delta: { role: 'assistant', content: 'chunk' },\n    };\n\n    for (let i = 0; i < 3; i++) {\n      if (this.shouldFail && i === this.failAfterChunks) {\n        throw new APIError('Mock LLM failed after chunks');\n      }\n      this.queue.put(chunk);\n      await delay(10);\n    }\n  }\n}\n\nclass MockLLM extends LLM {\n  shouldFail: boolean = false;\n  failAfterChunks: number = 0;\n  private _label: string;\n\n  constructor(label: string) {\n    super();\n    this._label = label;\n  }\n\n  label(): string {\n    return this._label;\n  }\n\n  chat(opts: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream {\n    return new MockLLMStream(\n      this,\n      {\n        chatCtx: opts.chatCtx,\n        toolCtx: opts.toolCtx,\n        connOptions: opts.connOptions!,\n      },\n      this.shouldFail,\n      this.failAfterChunks,\n    );\n  }\n}\n\ndescribe('FallbackAdapter', () => {\n  beforeAll(() => {\n    initializeLogger({ pretty: false });\n    // Suppress unhandled rejections from LLMStream background tasks\n    process.on('unhandledRejection', () => {});\n  });\n\n  it('should initialize correctly', () => {\n    const llm1 = new MockLLM('llm1');\n    const adapter = new FallbackAdapter({ llms: [llm1] });\n    expect(adapter.llms).toHaveLength(1);\n    expect(adapter.llms[0]).toBe(llm1);\n  });\n\n  it('should throw if no LLMs provided', () => {\n    expect(() => new FallbackAdapter({ llms: [] })).toThrow();\n  });\n\n  it('should use primary LLM if successful', async () => {\n    const llm1 = new MockLLM('llm1');\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    expect(chunks).toHaveLength(3);\n    // Should verify it used llm1 (we can check logs or spy, but simple success is good first step)\n  });\n\n  it('should fallback to second LLM if first fails immediately', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    expect(chunks).toHaveLength(3);\n    expect(adapter._status[0]!.available).toBe(false);\n    expect(adapter._status[1]!.available).toBe(true);\n  });\n\n  it('should fail if all LLMs fail', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    llm2.shouldFail = true;\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const errorPromise = new Promise<Error>((resolve) => {\n      adapter.on('error', (e) => resolve(e.error));\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    const error = await errorPromise;\n    expect(error).toBeInstanceOf(APIConnectionError);\n  });\n\n  it('should fail if chunks sent and retryOnChunkSent is false', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    llm1.failAfterChunks = 1; // Fail after 1 chunk\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({\n      llms: [llm1, llm2],\n      retryOnChunkSent: false,\n    });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const errorPromise = new Promise<Error>((resolve) => {\n      adapter.on('error', (e) => resolve(e.error));\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    const error = await errorPromise;\n    expect(error).toBeInstanceOf(APIError);\n  });\n\n  it('should fallback if chunks sent and retryOnChunkSent is true', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    llm1.failAfterChunks = 1;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({\n      llms: [llm1, llm2],\n      retryOnChunkSent: true,\n    });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    // 1 chunk from failed llm1 + 3 chunks from llm2\n    expect(chunks).toHaveLength(4);\n  });\n\n  it('should emit availability changed events', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const eventSpy = vi.fn();\n    (adapter as any).on('llm_availability_changed', eventSpy);\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    expect(eventSpy).toHaveBeenCalledWith(\n      expect.objectContaining({\n        llm: llm1,\n        available: false,\n      }),\n    );\n  });\n});\n"],"mappings":"AAGA,SAAS,WAAW,UAAU,QAAQ,IAAI,UAAU;AACpD,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,wBAAwB;AAEjC,SAAS,aAAa;AAEtB,SAAS,uBAAuB;AAChC,SAAyB,KAAK,iBAAiB;AAG/C,MAAM,sBAAsB,UAAU;AAAA,EAGpC,YACE,KACA,MAKQ,aAAsB,OACtB,kBAA0B,GAClC;AACA,UAAM,KAAK,IAAI;AAHP;AACA;AAGR,SAAK,QAAQ;AAAA,EACf;AAAA,EAdO;AAAA,EAgBP,MAAgB,MAAqB;AACnC,QAAI,KAAK,cAAc,KAAK,oBAAoB,GAAG;AACjD,YAAM,IAAI,SAAS,6BAA6B;AAAA,IAClD;AAEA,UAAM,QAAmB;AAAA,MACvB,IAAI;AAAA,MACJ,OAAO,EAAE,MAAM,aAAa,SAAS,QAAQ;AAAA,IAC/C;AAEA,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,UAAI,KAAK,cAAc,MAAM,KAAK,iBAAiB;AACjD,cAAM,IAAI,SAAS,8BAA8B;AAAA,MACnD;AACA,WAAK,MAAM,IAAI,KAAK;AACpB,YAAM,MAAM,EAAE;AAAA,IAChB;AAAA,EACF;AACF;AAEA,MAAM,gBAAgB,IAAI;AAAA,EACxB,aAAsB;AAAA,EACtB,kBAA0B;AAAA,EAClB;AAAA,EAER,YAAY,OAAe;AACzB,UAAM;AACN,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,QAAgB;AACd,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,KAAK,MAOS;AACZ,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,QACE,SAAS,KAAK;AAAA,QACd,SAAS,KAAK;AAAA,QACd,aAAa,KAAK;AAAA,MACpB;AAAA,MACA,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,MAAM;AAChC,YAAU,MAAM;AACd,qBAAiB,EAAE,QAAQ,MAAM,CAAC;AAElC,YAAQ,GAAG,sBAAsB,MAAM;AAAA,IAAC,CAAC;AAAA,EAC3C,CAAC;AAED,KAAG,+BAA+B,MAAM;AACtC,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AACpD,WAAO,QAAQ,IAAI,EAAE,aAAa,CAAC;AACnC,WAAO,QAAQ,KAAK,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACnC,CAAC;AAED,KAAG,oCAAoC,MAAM;AAC3C,WAAO,MAAM,IAAI,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ;AAAA,EAC1D,CAAC;AAED,KAAG,wCAAwC,YAAY;AACrD,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,MAAM,EAAE,aAAa,CAAC;AAAA,EAE/B,CAAC;AAED,KAAG,4DAA4D,YAAY;AACzE,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,MAAM,EAAE,aAAa,CAAC;AAC7B,WAAO,QAAQ,QAAQ,CAAC,EAAG,SAAS,EAAE,KAAK,KAAK;AAChD,WAAO,QAAQ,QAAQ,CAAC,EAAG,SAAS,EAAE,KAAK,IAAI;AAAA,EACjD,CAAC;AAED,KAAG,gCAAgC,YAAY;AAC7C,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,eAAe,IAAI,QAAe,CAAC,YAAY;AACnD,cAAQ,GAAG,SAAS,CAAC,MAAM,QAAQ,EAAE,KAAK,CAAC;AAAA,IAC7C,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,UAAM,QAAQ,MAAM;AACpB,WAAO,KAAK,EAAE,eAAe,kBAAkB;AAAA,EACjD,CAAC;AAED,KAAG,4DAA4D,YAAY;AACzE,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AACvB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB;AAAA,MAClC,MAAM,CAAC,MAAM,IAAI;AAAA,MACjB,kBAAkB;AAAA,IACpB,CAAC;AAED,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,eAAe,IAAI,QAAe,CAAC,YAAY;AACnD,cAAQ,GAAG,SAAS,CAAC,MAAM,QAAQ,EAAE,KAAK,CAAC;AAAA,IAC7C,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,UAAM,QAAQ,MAAM;AACpB,WAAO,KAAK,EAAE,eAAe,QAAQ;AAAA,EACvC,CAAC;AAED,KAAG,+DAA+D,YAAY;AAC5E,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AACvB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB;AAAA,MAClC,MAAM,CAAC,MAAM,IAAI;AAAA,MACjB,kBAAkB;AAAA,IACpB,CAAC;AAED,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAGA,WAAO,MAAM,EAAE,aAAa,CAAC;AAAA,EAC/B,CAAC;AAED,KAAG,2CAA2C,YAAY;AACxD,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,WAAW,GAAG,GAAG;AACvB,IAAC,QAAgB,GAAG,4BAA4B,QAAQ;AAExD,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,WAAO,QAAQ,EAAE;AAAA,MACf,OAAO,iBAAiB;AAAA,QACtB,KAAK;AAAA,QACL,WAAW;AAAA,MACb,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH,CAAC;","names":[]}

package/dist/llm/index.cjs

@@ -18,8 +18,10 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var llm_exports = {};
 __export(llm_exports, {
+  AgentHandoffItem: () => import_chat_context.AgentHandoffItem,
   ChatContext: () => import_chat_context.ChatContext,
   ChatMessage: () => import_chat_context.ChatMessage,
+  FallbackAdapter: () => import_fallback_adapter.FallbackAdapter,
   FunctionCall: () => import_chat_context.FunctionCall,
   FunctionCallOutput: () => import_chat_context.FunctionCallOutput,
   LLM: () => import_llm.LLM,
@@ -28,15 +30,18 @@ __export(llm_exports, {
   RealtimeSession: () => import_realtime.RealtimeSession,
   RemoteChatContext: () => import_remote_chat_context.RemoteChatContext,
   ToolError: () => import_tool_context.ToolError,
+  ToolFlag: () => import_tool_context.ToolFlag,
   computeChatCtxDiff: () => import_utils.computeChatCtxDiff,
   createAudioContent: () => import_chat_context.createAudioContent,
   createImageContent: () => import_chat_context.createImageContent,
   createToolOptions: () => import_utils.createToolOptions,
   executeToolCall: () => import_utils.executeToolCall,
+  formatChatHistory: () => import_utils.formatChatHistory,
   handoff: () => import_tool_context.handoff,
   isFunctionTool: () => import_tool_context.isFunctionTool,
   oaiBuildFunctionInfo: () => import_utils.oaiBuildFunctionInfo,
   oaiParams: () => import_utils.oaiParams,
+  serializeImage: () => import_utils.serializeImage,
   toJsonSchema: () => import_utils.toJsonSchema,
   tool: () => import_tool_context.tool
 });
@@ -47,10 +52,13 @@ var import_llm = require("./llm.cjs");
 var import_realtime = require("./realtime.cjs");
 var import_remote_chat_context = require("./remote_chat_context.cjs");
 var import_utils = require("./utils.cjs");
+var import_fallback_adapter = require("./fallback_adapter.cjs");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  AgentHandoffItem,
   ChatContext,
   ChatMessage,
+  FallbackAdapter,
   FunctionCall,
   FunctionCallOutput,
   LLM,
@@ -59,15 +67,18 @@ var import_utils = require("./utils.cjs");
   RealtimeSession,
   RemoteChatContext,
   ToolError,
+  ToolFlag,
   computeChatCtxDiff,
   createAudioContent,
   createImageContent,
   createToolOptions,
   executeToolCall,
+  formatChatHistory,
   handoff,
   isFunctionTool,
   oaiBuildFunctionInfo,
   oaiParams,
+  serializeImage,
   toJsonSchema,
   tool
 });

package/dist/llm/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAaO;AAEP,0BAYO;AAIP,iBAOO;AAEP,sBAWO;AAEP,iCAAkC;AAElC,mBAQO;","names":[]}
+{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  ToolFlag,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n  type MetricsReport,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  formatChatHistory,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  serializeImage,\n  toJsonSchema,\n  type FormatChatHistoryOptions,\n  type OpenAIFunctionParameters,\n  type SerializedImage,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAcO;AAEP,0BAcO;AAIP,iBAOO;AAEP,sBAWO;AAEP,iCAAkC;AAElC,mBAYO;AAEP,8BAIO;","names":[]}

package/dist/llm/index.d.cts

@@ -1,8 +1,9 @@
-export { handoff, isFunctionTool, tool, ToolError, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
-export { ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, } from './chat_context.js';
+export { handoff, isFunctionTool, tool, ToolError, ToolFlag, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
+export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, type MetricsReport, } from './chat_context.js';
 export type { ProviderFormat } from './provider_format/index.js';
 export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage, type LLMCallbacks, } from './llm.js';
 export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';
 export { RemoteChatContext } from './remote_chat_context.js';
-export { computeChatCtxDiff, createToolOptions, executeToolCall, oaiBuildFunctionInfo, oaiParams, toJsonSchema, type OpenAIFunctionParameters, } from './utils.js';
+export { computeChatCtxDiff, createToolOptions, executeToolCall, formatChatHistory, oaiBuildFunctionInfo, oaiParams, serializeImage, toJsonSchema, type FormatChatHistoryOptions, type OpenAIFunctionParameters, type SerializedImage, } from './utils.js';
+export { FallbackAdapter, type AvailabilityChangedEvent, type FallbackAdapterOptions, } from './fallback_adapter.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/llm/index.d.ts

@@ -1,8 +1,9 @@
-export { handoff, isFunctionTool, tool, ToolError, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
-export { ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, } from './chat_context.js';
+export { handoff, isFunctionTool, tool, ToolError, ToolFlag, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
+export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, type MetricsReport, } from './chat_context.js';
 export type { ProviderFormat } from './provider_format/index.js';
 export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage, type LLMCallbacks, } from './llm.js';
 export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';
 export { RemoteChatContext } from './remote_chat_context.js';
-export { computeChatCtxDiff, createToolOptions, executeToolCall, oaiBuildFunctionInfo, oaiParams, toJsonSchema, type OpenAIFunctionParameters, } from './utils.js';
+export { computeChatCtxDiff, createToolOptions, executeToolCall, formatChatHistory, oaiBuildFunctionInfo, oaiParams, serializeImage, toJsonSchema, type FormatChatHistoryOptions, type OpenAIFunctionParameters, type SerializedImage, } from './utils.js';
+export { FallbackAdapter, type AvailabilityChangedEvent, type FallbackAdapterOptions, } from './fallback_adapter.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,OAAO,EACP,cAAc,EACd,IAAI,EACJ,SAAS,EACT,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,QAAQ,GACd,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EACL,GAAG,EACH,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,aAAa,EACb,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,GACrC,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,SAAS,EACT,YAAY,EACZ,KAAK,wBAAwB,GAC9B,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,OAAO,EACP,cAAc,EACd,IAAI,EACJ,SAAS,EACT,QAAQ,EACR,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,QAAQ,GACd,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,aAAa,GACnB,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EACL,GAAG,EACH,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,aAAa,EACb,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,GACrC,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,EACjB,oBAAoB,EACpB,SAAS,EACT,cAAc,EACd,YAAY,EACZ,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAC7B,KAAK,eAAe,GACrB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,eAAe,EACf,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,uBAAuB,CAAC"}

package/dist/llm/index.js

@@ -2,9 +2,11 @@ import {
   handoff,
   isFunctionTool,
   tool,
-  ToolError
+  ToolError,
+  ToolFlag
 } from "./tool_context.js";
 import {
+  AgentHandoffItem,
   ChatContext,
   ChatMessage,
   createAudioContent,
@@ -25,13 +27,20 @@ import {
   computeChatCtxDiff,
   createToolOptions,
   executeToolCall,
+  formatChatHistory,
   oaiBuildFunctionInfo,
   oaiParams,
+  serializeImage,
   toJsonSchema
 } from "./utils.js";
+import {
+  FallbackAdapter
+} from "./fallback_adapter.js";
 export {
+  AgentHandoffItem,
   ChatContext,
   ChatMessage,
+  FallbackAdapter,
   FunctionCall,
   FunctionCallOutput,
   LLM,
@@ -40,15 +49,18 @@ export {
   RealtimeSession,
   RemoteChatContext,
   ToolError,
+  ToolFlag,
   computeChatCtxDiff,
   createAudioContent,
   createImageContent,
   createToolOptions,
   executeToolCall,
+  formatChatHistory,
   handoff,
   isFunctionTool,
   oaiBuildFunctionInfo,
   oaiParams,
+  serializeImage,
   toJsonSchema,
   tool
 };

package/dist/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OASK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAMK;AAIP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OASK;AAEP,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;","names":[]}
+{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  ToolFlag,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n  type MetricsReport,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  formatChatHistory,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  serializeImage,\n  toJsonSchema,\n  type FormatChatHistoryOptions,\n  type OpenAIFunctionParameters,\n  type SerializedImage,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OASK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAOK;AAIP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OASK;AAEP,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAIK;AAEP;AAAA,EACE;AAAA,OAGK;","names":[]}

package/dist/llm/llm.cjs

@@ -25,6 +25,8 @@ module.exports = __toCommonJS(llm_exports);
 var import_node_events = require("node:events");
 var import_exceptions = require("../_exceptions.cjs");
 var import_log = require("../log.cjs");
+var import_telemetry = require("../telemetry/index.cjs");
+var import_types = require("../types.cjs");
 var import_utils = require("../utils.cjs");
 var import_chat_context = require("./chat_context.cjs");
 class LLM extends import_node_events.EventEmitter {
@@ -42,6 +44,17 @@ class LLM extends import_node_events.EventEmitter {
   get model() {
     return "unknown";
   }
+  /**
+   * Get the provider name for this LLM instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider() {
+    return "unknown";
+  }
   /**
    * Pre-warm connection to the LLM service
    */
@@ -60,6 +73,7 @@ class LLMStream {
   #llm;
   #chatCtx;
   #toolCtx;
+  #llmRequestSpan;
   constructor(llm, {
     chatCtx,
     toolCtx,
@@ -74,15 +88,28 @@ class LLMStream {
       this.output.close();
       this.closed = true;
     });
-    (0, import_utils.startSoon)(() => this.mainTask().then(() => this.queue.close()));
+    (0, import_utils.startSoon)(() => this.mainTask().finally(() => this.queue.close()));
   }
-  async mainTask() {
+  _mainTaskImpl = async (span) => {
+    this.#llmRequestSpan = span;
+    span.setAttribute(import_telemetry.traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
-        return await this.run();
+        return await import_telemetry.tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(import_telemetry.traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              (0, import_telemetry.recordException)(attemptSpan, (0, import_utils.toError)(error));
+              throw error;
+            }
+          },
+          { name: "llm_request_run" }
+        );
       } catch (error) {
         if (error instanceof import_exceptions.APIError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+          const retryInterval = (0, import_types.intervalForRetry)(this._connOptions, i);
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
             throw error;
@@ -96,7 +123,7 @@ class LLMStream {
             this.emitError({ error, recoverable: true });
             this.logger.warn(
               { llm: this.#llm.label(), attempt: i + 1, error },
-              `failed to generate LLM completion, retrying in ${retryInterval}s`
+              `failed to generate LLM completion, retrying in ${retryInterval}ms`
             );
           }
           if (retryInterval > 0) {
@@ -108,7 +135,11 @@ class LLMStream {
         }
       }
     }
-  }
+  };
+  mainTask = async () => import_telemetry.tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+    name: "llm_request",
+    endOnExit: false
+  });
   emitError({ error, recoverable }) {
     this.#llm.emit("error", {
       type: "llm_error",
@@ -123,6 +154,7 @@ class LLMStream {
     let ttft = BigInt(-1);
     let requestId = "";
     let usage;
+    let completionStartTime;
     for await (const ev of this.queue) {
       if (this.abortController.signal.aborted) {
         break;
@@ -131,6 +163,7 @@ class LLMStream {
       requestId = ev.id;
       if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
+        completionStartTime = (/* @__PURE__ */ new Date()).toISOString();
       }
       if (ev.usage) {
         usage = ev.usage;
@@ -138,12 +171,13 @@ class LLMStream {
     }
     this.output.close();
     const duration = process.hrtime.bigint() - startTime;
+    const durationMs = Math.trunc(Number(duration / BigInt(1e6)));
     const metrics = {
       type: "llm_metrics",
       timestamp: Date.now(),
       requestId,
-      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1e6))),
-      duration: Math.trunc(Number(duration / BigInt(1e6))),
+      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1e6))),
+      durationMs,
       cancelled: this.abortController.signal.aborted,
       label: this.#llm.label(),
       completionTokens: (usage == null ? void 0 : usage.completionTokens) || 0,
@@ -151,10 +185,30 @@ class LLMStream {
       promptCachedTokens: (usage == null ? void 0 : usage.promptCachedTokens) || 0,
       totalTokens: (usage == null ? void 0 : usage.totalTokens) || 0,
       tokensPerSecond: (() => {
-        const durationSeconds = Math.trunc(Number(duration / BigInt(1e9)));
-        return durationSeconds > 0 ? ((usage == null ? void 0 : usage.completionTokens) || 0) / durationSeconds : 0;
-      })()
+        if (durationMs <= 0) {
+          return 0;
+        }
+        return ((usage == null ? void 0 : usage.completionTokens) || 0) / (durationMs / 1e3);
+      })(),
+      metadata: {
+        modelProvider: this.#llm.provider,
+        modelName: this.#llm.model
+      }
     };
+    if (this.#llmRequestSpan) {
+      this.#llmRequestSpan.setAttribute(import_telemetry.traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));
+      this.#llmRequestSpan.setAttributes({
+        [import_telemetry.traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,
+        [import_telemetry.traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens
+      });
+      if (completionStartTime) {
+        this.#llmRequestSpan.setAttribute(
+          import_telemetry.traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,
+          completionStartTime
+        );
+      }
+      this.#llmRequestSpan.end();
+    }
     this.#llm.emit("metrics_collected", metrics);
   }
   /** The function context of this stream. */

package/dist/llm/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, any>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        const durationSeconds = Math.trunc(Number(duration / BigInt(1000000000)));\n        return durationSeconds > 0 ? (usage?.completionTokens || 0) / durationSeconds : 0;\n      })(),\n    };\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAGpB,mBAA8D;AAC9D,0BAAmE;AAmC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,cAAM,kBAAkB,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AACxE,eAAO,kBAAkB,MAAK,+BAAO,qBAAoB,KAAK,kBAAkB;AAAA,MAClF,GAAG;AAAA,IACL;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n  extra?: Record<string, unknown>;\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Get the provider name for this LLM instance.\n   *\n   * @returns The provider name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their provider information.\n   */\n  get provider(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}ms`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n      metadata: {\n        modelProvider: this.#llm.provider,\n        modelName: this.#llm.model,\n      },\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAEpB,uBAAoD;AACpD,mBAAyD;AACzD,mBAA8D;AAC9D,0BAAmE;AAoC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,4BAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,wBAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,4BAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,oDAAgB,iBAAa,sBAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,oBAAgB,+BAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,wBAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,MACH,UAAU;AAAA,QACR,eAAe,KAAK,KAAK;AAAA,QACzB,WAAW,KAAK,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,4BAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,4BAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,4BAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,4BAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/llm/llm.d.cts

@@ -1,6 +1,6 @@
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { LLMMetrics } from '../metrics/base.js';
-import type { APIConnectOptions } from '../types.js';
+import { type APIConnectOptions } from '../types.js';
 import { AsyncIterableQueue } from '../utils.js';
 import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
 import type { ToolChoice, ToolContext } from './tool_context.js';
@@ -8,6 +8,7 @@ export interface ChoiceDelta {
     role: ChatRole;
     content?: string;
     toolCalls?: FunctionCall[];
+    extra?: Record<string, unknown>;
 }
 export interface CompletionUsage {
     completionTokens: number;
@@ -44,6 +45,15 @@ export declare abstract class LLM extends LLM_base {
      * Plugins should override this property to provide their model information.
      */
     get model(): string;
+    /**
+     * Get the provider name for this LLM instance.
+     *
+     * @returns The provider name if available, "unknown" otherwise.
+     *
+     * @remarks
+     * Plugins should override this property to provide their provider information.
+     */
+    get provider(): string;
     /**
      * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
      */
@@ -53,7 +63,7 @@ export declare abstract class LLM extends LLM_base {
         connOptions?: APIConnectOptions;
         parallelToolCalls?: boolean;
         toolChoice?: ToolChoice;
-        extraKwargs?: Record<string, any>;
+        extraKwargs?: Record<string, unknown>;
     }): LLMStream;
     /**
      * Pre-warm connection to the LLM service
@@ -74,6 +84,7 @@ export declare abstract class LLMStream implements AsyncIterableIterator<ChatChu
         toolCtx?: ToolContext;
         connOptions: APIConnectOptions;
     });
+    private _mainTaskImpl;
     private mainTask;
     private emitError;
     protected monitorMetrics(): Promise<void>;