@livekit/agents 1.0.32 → 1.0.34

package/dist/llm/fallback_adapter.test.js

@@ -0,0 +1,175 @@
+import { beforeAll, describe, expect, it, vi } from "vitest";
+import { APIConnectionError, APIError } from "../_exceptions.js";
+import { initializeLogger } from "../log.js";
+import { delay } from "../utils.js";
+import { FallbackAdapter } from "./fallback_adapter.js";
+import { LLM, LLMStream } from "./llm.js";
+class MockLLMStream extends LLMStream {
+  constructor(llm, opts, shouldFail = false, failAfterChunks = 0) {
+    super(llm, opts);
+    this.shouldFail = shouldFail;
+    this.failAfterChunks = failAfterChunks;
+    this.myLLM = llm;
+  }
+  myLLM;
+  async run() {
+    if (this.shouldFail && this.failAfterChunks === 0) {
+      throw new APIError("Mock LLM failed immediately");
+    }
+    const chunk = {
+      id: "test-id",
+      delta: { role: "assistant", content: "chunk" }
+    };
+    for (let i = 0; i < 3; i++) {
+      if (this.shouldFail && i === this.failAfterChunks) {
+        throw new APIError("Mock LLM failed after chunks");
+      }
+      this.queue.put(chunk);
+      await delay(10);
+    }
+  }
+}
+class MockLLM extends LLM {
+  shouldFail = false;
+  failAfterChunks = 0;
+  _label;
+  constructor(label) {
+    super();
+    this._label = label;
+  }
+  label() {
+    return this._label;
+  }
+  chat(opts) {
+    return new MockLLMStream(
+      this,
+      {
+        chatCtx: opts.chatCtx,
+        toolCtx: opts.toolCtx,
+        connOptions: opts.connOptions
+      },
+      this.shouldFail,
+      this.failAfterChunks
+    );
+  }
+}
+describe("FallbackAdapter", () => {
+  beforeAll(() => {
+    initializeLogger({ pretty: false });
+    process.on("unhandledRejection", () => {
+    });
+  });
+  it("should initialize correctly", () => {
+    const llm1 = new MockLLM("llm1");
+    const adapter = new FallbackAdapter({ llms: [llm1] });
+    expect(adapter.llms).toHaveLength(1);
+    expect(adapter.llms[0]).toBe(llm1);
+  });
+  it("should throw if no LLMs provided", () => {
+    expect(() => new FallbackAdapter({ llms: [] })).toThrow();
+  });
+  it("should use primary LLM if successful", async () => {
+    const llm1 = new MockLLM("llm1");
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    expect(chunks).toHaveLength(3);
+  });
+  it("should fallback to second LLM if first fails immediately", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    expect(chunks).toHaveLength(3);
+    expect(adapter._status[0].available).toBe(false);
+    expect(adapter._status[1].available).toBe(true);
+  });
+  it("should fail if all LLMs fail", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM("llm2");
+    llm2.shouldFail = true;
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const errorPromise = new Promise((resolve) => {
+      adapter.on("error", (e) => resolve(e.error));
+    });
+    for await (const _ of stream) {
+    }
+    const error = await errorPromise;
+    expect(error).toBeInstanceOf(APIConnectionError);
+  });
+  it("should fail if chunks sent and retryOnChunkSent is false", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    llm1.failAfterChunks = 1;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({
+      llms: [llm1, llm2],
+      retryOnChunkSent: false
+    });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const errorPromise = new Promise((resolve) => {
+      adapter.on("error", (e) => resolve(e.error));
+    });
+    for await (const _ of stream) {
+    }
+    const error = await errorPromise;
+    expect(error).toBeInstanceOf(APIError);
+  });
+  it("should fallback if chunks sent and retryOnChunkSent is true", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    llm1.failAfterChunks = 1;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({
+      llms: [llm1, llm2],
+      retryOnChunkSent: true
+    });
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    const chunks = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+    expect(chunks).toHaveLength(4);
+  });
+  it("should emit availability changed events", async () => {
+    const llm1 = new MockLLM("llm1");
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM("llm2");
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+    const eventSpy = vi.fn();
+    adapter.on("llm_availability_changed", eventSpy);
+    const stream = adapter.chat({
+      chatCtx: {}
+    });
+    for await (const _ of stream) {
+    }
+    expect(eventSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        llm: llm1,
+        available: false
+      })
+    );
+  });
+});
+//# sourceMappingURL=fallback_adapter.test.js.map

package/dist/llm/fallback_adapter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/llm/fallback_adapter.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { beforeAll, describe, expect, it, vi } from 'vitest';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { initializeLogger } from '../log.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { delay } from '../utils.js';\nimport type { ChatContext } from './chat_context.js';\nimport { FallbackAdapter } from './fallback_adapter.js';\nimport { type ChatChunk, LLM, LLMStream } from './llm.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nclass MockLLMStream extends LLMStream {\n  public myLLM: LLM;\n\n  constructor(\n    llm: LLM,\n    opts: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n    private shouldFail: boolean = false,\n    private failAfterChunks: number = 0,\n  ) {\n    super(llm, opts);\n    this.myLLM = llm;\n  }\n\n  protected async run(): Promise<void> {\n    if (this.shouldFail && this.failAfterChunks === 0) {\n      throw new APIError('Mock LLM failed immediately');\n    }\n\n    const chunk: ChatChunk = {\n      id: 'test-id',\n      delta: { role: 'assistant', content: 'chunk' },\n    };\n\n    for (let i = 0; i < 3; i++) {\n      if (this.shouldFail && i === this.failAfterChunks) {\n        throw new APIError('Mock LLM failed after chunks');\n      }\n      this.queue.put(chunk);\n      await delay(10);\n    }\n  }\n}\n\nclass MockLLM extends LLM {\n  shouldFail: boolean = false;\n  failAfterChunks: number = 0;\n  private _label: string;\n\n  constructor(label: string) {\n    super();\n    this._label = label;\n  }\n\n  label(): string {\n    return this._label;\n  }\n\n  chat(opts: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream {\n    return new MockLLMStream(\n      this,\n      {\n        chatCtx: opts.chatCtx,\n        toolCtx: opts.toolCtx,\n        connOptions: opts.connOptions!,\n      },\n      this.shouldFail,\n      this.failAfterChunks,\n    );\n  }\n}\n\ndescribe('FallbackAdapter', () => {\n  beforeAll(() => {\n    initializeLogger({ pretty: false });\n    // Suppress unhandled rejections from LLMStream background tasks\n    process.on('unhandledRejection', () => {});\n  });\n\n  it('should initialize correctly', () => {\n    const llm1 = new MockLLM('llm1');\n    const adapter = new FallbackAdapter({ llms: [llm1] });\n    expect(adapter.llms).toHaveLength(1);\n    expect(adapter.llms[0]).toBe(llm1);\n  });\n\n  it('should throw if no LLMs provided', () => {\n    expect(() => new FallbackAdapter({ llms: [] })).toThrow();\n  });\n\n  it('should use primary LLM if successful', async () => {\n    const llm1 = new MockLLM('llm1');\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    expect(chunks).toHaveLength(3);\n    // Should verify it used llm1 (we can check logs or spy, but simple success is good first step)\n  });\n\n  it('should fallback to second LLM if first fails immediately', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    expect(chunks).toHaveLength(3);\n    expect(adapter._status[0]!.available).toBe(false);\n    expect(adapter._status[1]!.available).toBe(true);\n  });\n\n  it('should fail if all LLMs fail', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    llm2.shouldFail = true;\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const errorPromise = new Promise<Error>((resolve) => {\n      adapter.on('error', (e) => resolve(e.error));\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    const error = await errorPromise;\n    expect(error).toBeInstanceOf(APIConnectionError);\n  });\n\n  it('should fail if chunks sent and retryOnChunkSent is false', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    llm1.failAfterChunks = 1; // Fail after 1 chunk\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({\n      llms: [llm1, llm2],\n      retryOnChunkSent: false,\n    });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const errorPromise = new Promise<Error>((resolve) => {\n      adapter.on('error', (e) => resolve(e.error));\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    const error = await errorPromise;\n    expect(error).toBeInstanceOf(APIError);\n  });\n\n  it('should fallback if chunks sent and retryOnChunkSent is true', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    llm1.failAfterChunks = 1;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({\n      llms: [llm1, llm2],\n      retryOnChunkSent: true,\n    });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    // 1 chunk from failed llm1 + 3 chunks from llm2\n    expect(chunks).toHaveLength(4);\n  });\n\n  it('should emit availability changed events', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const eventSpy = vi.fn();\n    (adapter as any).on('llm_availability_changed', eventSpy);\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    expect(eventSpy).toHaveBeenCalledWith(\n      expect.objectContaining({\n        llm: llm1,\n        available: false,\n      }),\n    );\n  });\n});\n"],"mappings":"AAGA,SAAS,WAAW,UAAU,QAAQ,IAAI,UAAU;AACpD,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,wBAAwB;AAEjC,SAAS,aAAa;AAEtB,SAAS,uBAAuB;AAChC,SAAyB,KAAK,iBAAiB;AAG/C,MAAM,sBAAsB,UAAU;AAAA,EAGpC,YACE,KACA,MAKQ,aAAsB,OACtB,kBAA0B,GAClC;AACA,UAAM,KAAK,IAAI;AAHP;AACA;AAGR,SAAK,QAAQ;AAAA,EACf;AAAA,EAdO;AAAA,EAgBP,MAAgB,MAAqB;AACnC,QAAI,KAAK,cAAc,KAAK,oBAAoB,GAAG;AACjD,YAAM,IAAI,SAAS,6BAA6B;AAAA,IAClD;AAEA,UAAM,QAAmB;AAAA,MACvB,IAAI;AAAA,MACJ,OAAO,EAAE,MAAM,aAAa,SAAS,QAAQ;AAAA,IAC/C;AAEA,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,UAAI,KAAK,cAAc,MAAM,KAAK,iBAAiB;AACjD,cAAM,IAAI,SAAS,8BAA8B;AAAA,MACnD;AACA,WAAK,MAAM,IAAI,KAAK;AACpB,YAAM,MAAM,EAAE;AAAA,IAChB;AAAA,EACF;AACF;AAEA,MAAM,gBAAgB,IAAI;AAAA,EACxB,aAAsB;AAAA,EACtB,kBAA0B;AAAA,EAClB;AAAA,EAER,YAAY,OAAe;AACzB,UAAM;AACN,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,QAAgB;AACd,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,KAAK,MAOS;AACZ,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,QACE,SAAS,KAAK;AAAA,QACd,SAAS,KAAK;AAAA,QACd,aAAa,KAAK;AAAA,MACpB;AAAA,MACA,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,MAAM;AAChC,YAAU,MAAM;AACd,qBAAiB,EAAE,QAAQ,MAAM,CAAC;AAElC,YAAQ,GAAG,sBAAsB,MAAM;AAAA,IAAC,CAAC;AAAA,EAC3C,CAAC;AAED,KAAG,+BAA+B,MAAM;AACtC,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AACpD,WAAO,QAAQ,IAAI,EAAE,aAAa,CAAC;AACnC,WAAO,QAAQ,KAAK,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACnC,CAAC;AAED,KAAG,oCAAoC,MAAM;AAC3C,WAAO,MAAM,IAAI,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ;AAAA,EAC1D,CAAC;AAED,KAAG,wCAAwC,YAAY;AACrD,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,MAAM,EAAE,aAAa,CAAC;AAAA,EAE/B,CAAC;AAED,KAAG,4DAA4D,YAAY;AACzE,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,MAAM,EAAE,aAAa,CAAC;AAC7B,WAAO,QAAQ,QAAQ,CAAC,EAAG,SAAS,EAAE,KAAK,KAAK;AAChD,WAAO,QAAQ,QAAQ,CAAC,EAAG,SAAS,EAAE,KAAK,IAAI;AAAA,EACjD,CAAC;AAED,KAAG,gCAAgC,YAAY;AAC7C,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,eAAe,IAAI,QAAe,CAAC,YAAY;AACnD,cAAQ,GAAG,SAAS,CAAC,MAAM,QAAQ,EAAE,KAAK,CAAC;AAAA,IAC7C,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,UAAM,QAAQ,MAAM;AACpB,WAAO,KAAK,EAAE,eAAe,kBAAkB;AAAA,EACjD,CAAC;AAED,KAAG,4DAA4D,YAAY;AACzE,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AACvB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB;AAAA,MAClC,MAAM,CAAC,MAAM,IAAI;AAAA,MACjB,kBAAkB;AAAA,IACpB,CAAC;AAED,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,eAAe,IAAI,QAAe,CAAC,YAAY;AACnD,cAAQ,GAAG,SAAS,CAAC,MAAM,QAAQ,EAAE,KAAK,CAAC;AAAA,IAC7C,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,UAAM,QAAQ,MAAM;AACpB,WAAO,KAAK,EAAE,eAAe,QAAQ;AAAA,EACvC,CAAC;AAED,KAAG,+DAA+D,YAAY;AAC5E,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AACvB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB;AAAA,MAClC,MAAM,CAAC,MAAM,IAAI;AAAA,MACjB,kBAAkB;AAAA,IACpB,CAAC;AAED,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAGA,WAAO,MAAM,EAAE,aAAa,CAAC;AAAA,EAC/B,CAAC;AAED,KAAG,2CAA2C,YAAY;AACxD,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,WAAW,GAAG,GAAG;AACvB,IAAC,QAAgB,GAAG,4BAA4B,QAAQ;AAExD,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,WAAO,QAAQ,EAAE;AAAA,MACf,OAAO,iBAAiB;AAAA,QACtB,KAAK;AAAA,QACL,WAAW;AAAA,MACb,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH,CAAC;","names":[]}

package/dist/llm/index.cjs

@@ -21,6 +21,7 @@ __export(llm_exports, {
   AgentHandoffItem: () => import_chat_context.AgentHandoffItem,
   ChatContext: () => import_chat_context.ChatContext,
   ChatMessage: () => import_chat_context.ChatMessage,
+  FallbackAdapter: () => import_fallback_adapter.FallbackAdapter,
   FunctionCall: () => import_chat_context.FunctionCall,
   FunctionCallOutput: () => import_chat_context.FunctionCallOutput,
   LLM: () => import_llm.LLM,
@@ -48,11 +49,13 @@ var import_llm = require("./llm.cjs");
 var import_realtime = require("./realtime.cjs");
 var import_remote_chat_context = require("./remote_chat_context.cjs");
 var import_utils = require("./utils.cjs");
+var import_fallback_adapter = require("./fallback_adapter.cjs");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AgentHandoffItem,
   ChatContext,
   ChatMessage,
+  FallbackAdapter,
   FunctionCall,
   FunctionCallOutput,
   LLM,

package/dist/llm/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAaO;AAEP,0BAaO;AAIP,iBAOO;AAEP,sBAWO;AAEP,iCAAkC;AAElC,mBAQO;","names":[]}
+{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAaO;AAEP,0BAaO;AAIP,iBAOO;AAEP,sBAWO;AAEP,iCAAkC;AAElC,mBAQO;AAEP,8BAIO;","names":[]}

package/dist/llm/index.d.cts

@@ -5,4 +5,5 @@ export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage,
 export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';
 export { RemoteChatContext } from './remote_chat_context.js';
 export { computeChatCtxDiff, createToolOptions, executeToolCall, oaiBuildFunctionInfo, oaiParams, toJsonSchema, type OpenAIFunctionParameters, } from './utils.js';
+export { FallbackAdapter, type AvailabilityChangedEvent, type FallbackAdapterOptions, } from './fallback_adapter.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/llm/index.d.ts

@@ -5,4 +5,5 @@ export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage,
 export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';
 export { RemoteChatContext } from './remote_chat_context.js';
 export { computeChatCtxDiff, createToolOptions, executeToolCall, oaiBuildFunctionInfo, oaiParams, toJsonSchema, type OpenAIFunctionParameters, } from './utils.js';
+export { FallbackAdapter, type AvailabilityChangedEvent, type FallbackAdapterOptions, } from './fallback_adapter.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,OAAO,EACP,cAAc,EACd,IAAI,EACJ,SAAS,EACT,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,QAAQ,GACd,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EACL,GAAG,EACH,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,aAAa,EACb,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,GACrC,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,SAAS,EACT,YAAY,EACZ,KAAK,wBAAwB,GAC9B,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,OAAO,EACP,cAAc,EACd,IAAI,EACJ,SAAS,EACT,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,QAAQ,GACd,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EACL,GAAG,EACH,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,aAAa,EACb,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,GACrC,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,SAAS,EACT,YAAY,EACZ,KAAK,wBAAwB,GAC9B,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,eAAe,EACf,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,uBAAuB,CAAC"}

package/dist/llm/index.js

@@ -30,10 +30,14 @@ import {
   oaiParams,
   toJsonSchema
 } from "./utils.js";
+import {
+  FallbackAdapter
+} from "./fallback_adapter.js";
 export {
   AgentHandoffItem,
   ChatContext,
   ChatMessage,
+  FallbackAdapter,
   FunctionCall,
   FunctionCallOutput,
   LLM,

package/dist/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OASK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAMK;AAIP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OASK;AAEP,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;","names":[]}
+{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OASK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAMK;AAIP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OASK;AAEP,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,OAGK;","names":[]}

package/dist/llm/llm.cjs

@@ -77,7 +77,7 @@ class LLMStream {
       this.output.close();
       this.closed = true;
     });
-    (0, import_utils.startSoon)(() => this.mainTask().then(() => this.queue.close()));
+    (0, import_utils.startSoon)(() => this.mainTask().finally(() => this.queue.close()));
   }
   _mainTaskImpl = async (span) => {
     this.#llmRequestSpan = span;

package/dist/llm/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAEpB,uBAAoD;AACpD,mBAAyD;AACzD,mBAA8D;AAC9D,0BAAmE;AAmC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,4BAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,wBAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,4BAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,oDAAgB,iBAAa,sBAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,oBAAgB,+BAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,wBAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,4BAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,4BAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,4BAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,4BAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAEpB,uBAAoD;AACpD,mBAAyD;AACzD,mBAA8D;AAC9D,0BAAmE;AAmC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,4BAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,wBAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,4BAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,oDAAgB,iBAAa,sBAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,oBAAgB,+BAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,wBAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,4BAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,4BAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,4BAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,4BAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/llm/llm.js

@@ -53,7 +53,7 @@ class LLMStream {
       this.output.close();
       this.closed = true;
     });
-    startSoon(() => this.mainTask().then(() => this.queue.close()));
+    startSoon(() => this.mainTask().finally(() => this.queue.close()));
   }
   _mainTaskImpl = async (span) => {
     this.#llmRequestSpan = span;

package/dist/llm/llm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAC7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,WAAW;AAEpB,SAAS,iBAAiB,YAAY,cAAc;AACpD,SAAiC,wBAAwB;AACzD,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAC9D,eAAmE;AAmC5D,MAAe,YAAa,aAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,SAAS,IAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,WAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,OAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,WAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,8BAAgB,aAAa,QAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,iBAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,OAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,WAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,WAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,WAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,WAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAC7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,WAAW;AAEpB,SAAS,iBAAiB,YAAY,cAAc;AACpD,SAAiC,wBAAwB;AACzD,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAC9D,eAAmE;AAmC5D,MAAe,YAAa,aAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,SAAS,IAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,WAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,OAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,WAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,8BAAgB,aAAa,QAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,iBAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,OAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,WAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,WAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,WAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,WAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/log.cjs

@@ -64,12 +64,12 @@ const enableOtelLogging = () => {
   }
   otelEnabled = true;
   const { pretty, level } = loggerOptions;
-  const terminalLevel = level || "info";
+  const logLevel = level || "info";
   const streams = [
-    { stream: pretty ? (0, import_pino_pretty.build)({ colorize: true }) : process.stdout, level: terminalLevel },
+    { stream: pretty ? (0, import_pino_pretty.build)({ colorize: true }) : process.stdout, level: logLevel },
     { stream: new OtelDestination(), level: "debug" }
   ];
-  logger = (0, import_pino.pino)({ level: "debug" }, (0, import_pino.multistream)(streams));
+  logger = (0, import_pino.pino)({ level: logLevel }, (0, import_pino.multistream)(streams));
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/log.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/log.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Writable } from 'node:stream';\nimport type { DestinationStream, Logger } from 'pino';\nimport { multistream, pino } from 'pino';\nimport { build as pinoPretty } from 'pino-pretty';\nimport { type PinoLogObject, emitToOtel } from './telemetry/pino_otel_transport.js';\n\n/** @internal */\nexport type LoggerOptions = {\n  pretty: boolean;\n  level?: string;\n};\n\n/** @internal */\nexport let loggerOptions: LoggerOptions;\n\n/** @internal */\nlet logger: Logger | undefined = undefined;\n\n/** @internal */\nlet otelEnabled = false;\n\n/** @internal */\nexport const log = () => {\n  if (!logger) {\n    throw new TypeError('logger not initialized. did you forget to run initializeLogger()?');\n  }\n  return logger;\n};\n\n/** @internal */\nexport const initializeLogger = ({ pretty, level }: LoggerOptions) => {\n  loggerOptions = { pretty, level };\n  logger = pino(\n    { level: level || 'info' },\n    pretty ? pinoPretty({ colorize: true }) : process.stdout,\n  );\n};\n\n/**\n * Custom Pino destination that parses JSON logs and emits to OTEL.\n * This receives the FULL serialized log including msg, level, time, etc.\n */\nclass OtelDestination extends Writable {\n  _write(chunk: Buffer, _encoding: string, callback: (error?: Error | null) => void): void {\n    try {\n      const line = chunk.toString().trim();\n      if (line) {\n        const logObj = JSON.parse(line) as PinoLogObject;\n        emitToOtel(logObj);\n      }\n    } catch {\n      // Ignore parse errors (e.g., non-JSON lines)\n    }\n    callback();\n  }\n}\n\n/**\n * Enable OTEL logging by reconfiguring the logger with multistream.\n * Uses a custom destination that receives full JSON logs (with msg, level, time).\n *\n * The base logger level is set to 'debug' so all logs are generated,\n * while each stream filters to its own level:\n * - Terminal: user-specified level (default: 'info')\n * - OTEL/Cloud: always 'debug' to capture all logs for observability\n *\n * @internal\n */\nexport const enableOtelLogging = () => {\n  if (otelEnabled || !logger) {\n    console.warn('OTEL logging already enabled or logger not initialized');\n    return;\n  }\n  otelEnabled = true;\n\n  const { pretty, level } = loggerOptions;\n\n  const terminalLevel = level || 'info';\n  const streams: { stream: DestinationStream; level: string }[] = [\n    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: terminalLevel },\n    { stream: new OtelDestination(), level: 'debug' },\n  ];\n\n  // Base level must be 'debug' to generate all logs; each stream filters independently\n  logger = pino({ level: 'debug' }, multistream(streams));\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,yBAAyB;AAEzB,kBAAkC;AAClC,yBAAoC;AACpC,iCAA+C;AASxC,IAAI;AAGX,IAAI,SAA6B;AAGjC,IAAI,cAAc;AAGX,MAAM,MAAM,MAAM;AACvB,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,UAAU,mEAAmE;AAAA,EACzF;AACA,SAAO;AACT;AAGO,MAAM,mBAAmB,CAAC,EAAE,QAAQ,MAAM,MAAqB;AACpE,kBAAgB,EAAE,QAAQ,MAAM;AAChC,eAAS;AAAA,IACP,EAAE,OAAO,SAAS,OAAO;AAAA,IACzB,aAAS,mBAAAA,OAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ;AAAA,EACpD;AACF;AAMA,MAAM,wBAAwB,4BAAS;AAAA,EACrC,OAAO,OAAe,WAAmB,UAAgD;AACvF,QAAI;AACF,YAAM,OAAO,MAAM,SAAS,EAAE,KAAK;AACnC,UAAI,MAAM;AACR,cAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,mDAAW,MAAM;AAAA,MACnB;AAAA,IACF,QAAQ;AAAA,IAER;AACA,aAAS;AAAA,EACX;AACF;AAaO,MAAM,oBAAoB,MAAM;AACrC,MAAI,eAAe,CAAC,QAAQ;AAC1B,YAAQ,KAAK,wDAAwD;AACrE;AAAA,EACF;AACA,gBAAc;AAEd,QAAM,EAAE,QAAQ,MAAM,IAAI;AAE1B,QAAM,gBAAgB,SAAS;AAC/B,QAAM,UAA0D;AAAA,IAC9D,EAAE,QAAQ,aAAS,mBAAAA,OAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ,QAAQ,OAAO,cAAc;AAAA,IACzF,EAAE,QAAQ,IAAI,gBAAgB,GAAG,OAAO,QAAQ;AAAA,EAClD;AAGA,eAAS,kBAAK,EAAE,OAAO,QAAQ,OAAG,yBAAY,OAAO,CAAC;AACxD;","names":["pinoPretty"]}
+{"version":3,"sources":["../src/log.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Writable } from 'node:stream';\nimport type { DestinationStream, Logger } from 'pino';\nimport { multistream, pino } from 'pino';\nimport { build as pinoPretty } from 'pino-pretty';\nimport { type PinoLogObject, emitToOtel } from './telemetry/pino_otel_transport.js';\n\n/** @internal */\nexport type LoggerOptions = {\n  pretty: boolean;\n  level?: string;\n};\n\n/** @internal */\nexport let loggerOptions: LoggerOptions;\n\n/** @internal */\nlet logger: Logger | undefined = undefined;\n\n/** @internal */\nlet otelEnabled = false;\n\n/** @internal */\nexport const log = () => {\n  if (!logger) {\n    throw new TypeError('logger not initialized. did you forget to run initializeLogger()?');\n  }\n  return logger;\n};\n\n/** @internal */\nexport const initializeLogger = ({ pretty, level }: LoggerOptions) => {\n  loggerOptions = { pretty, level };\n  logger = pino(\n    { level: level || 'info' },\n    pretty ? pinoPretty({ colorize: true }) : process.stdout,\n  );\n};\n\n/**\n * Custom Pino destination that parses JSON logs and emits to OTEL.\n * This receives the FULL serialized log including msg, level, time, etc.\n */\nclass OtelDestination extends Writable {\n  _write(chunk: Buffer, _encoding: string, callback: (error?: Error | null) => void): void {\n    try {\n      const line = chunk.toString().trim();\n      if (line) {\n        const logObj = JSON.parse(line) as PinoLogObject;\n        emitToOtel(logObj);\n      }\n    } catch {\n      // Ignore parse errors (e.g., non-JSON lines)\n    }\n    callback();\n  }\n}\n\n/**\n * Enable OTEL logging by reconfiguring the logger with multistream.\n * Uses a custom destination that receives full JSON logs (with msg, level, time).\n *\n * @internal\n */\nexport const enableOtelLogging = () => {\n  if (otelEnabled || !logger) {\n    console.warn('OTEL logging already enabled or logger not initialized');\n    return;\n  }\n  otelEnabled = true;\n\n  const { pretty, level } = loggerOptions;\n\n  const logLevel = level || 'info';\n  const streams: { stream: DestinationStream; level: string }[] = [\n    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: logLevel },\n    { stream: new OtelDestination(), level: 'debug' },\n  ];\n\n  logger = pino({ level: logLevel }, multistream(streams));\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,yBAAyB;AAEzB,kBAAkC;AAClC,yBAAoC;AACpC,iCAA+C;AASxC,IAAI;AAGX,IAAI,SAA6B;AAGjC,IAAI,cAAc;AAGX,MAAM,MAAM,MAAM;AACvB,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,UAAU,mEAAmE;AAAA,EACzF;AACA,SAAO;AACT;AAGO,MAAM,mBAAmB,CAAC,EAAE,QAAQ,MAAM,MAAqB;AACpE,kBAAgB,EAAE,QAAQ,MAAM;AAChC,eAAS;AAAA,IACP,EAAE,OAAO,SAAS,OAAO;AAAA,IACzB,aAAS,mBAAAA,OAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ;AAAA,EACpD;AACF;AAMA,MAAM,wBAAwB,4BAAS;AAAA,EACrC,OAAO,OAAe,WAAmB,UAAgD;AACvF,QAAI;AACF,YAAM,OAAO,MAAM,SAAS,EAAE,KAAK;AACnC,UAAI,MAAM;AACR,cAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,mDAAW,MAAM;AAAA,MACnB;AAAA,IACF,QAAQ;AAAA,IAER;AACA,aAAS;AAAA,EACX;AACF;AAQO,MAAM,oBAAoB,MAAM;AACrC,MAAI,eAAe,CAAC,QAAQ;AAC1B,YAAQ,KAAK,wDAAwD;AACrE;AAAA,EACF;AACA,gBAAc;AAEd,QAAM,EAAE,QAAQ,MAAM,IAAI;AAE1B,QAAM,WAAW,SAAS;AAC1B,QAAM,UAA0D;AAAA,IAC9D,EAAE,QAAQ,aAAS,mBAAAA,OAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ,QAAQ,OAAO,SAAS;AAAA,IACpF,EAAE,QAAQ,IAAI,gBAAgB,GAAG,OAAO,QAAQ;AAAA,EAClD;AAEA,eAAS,kBAAK,EAAE,OAAO,SAAS,OAAG,yBAAY,OAAO,CAAC;AACzD;","names":["pinoPretty"]}

package/dist/log.d.cts

@@ -14,11 +14,6 @@ export declare const initializeLogger: ({ pretty, level }: LoggerOptions) => voi
  * Enable OTEL logging by reconfiguring the logger with multistream.
  * Uses a custom destination that receives full JSON logs (with msg, level, time).
  *
- * The base logger level is set to 'debug' so all logs are generated,
- * while each stream filters to its own level:
- * - Terminal: user-specified level (default: 'info')
- * - OTEL/Cloud: always 'debug' to capture all logs for observability
- *
  * @internal
  */
 export declare const enableOtelLogging: () => void;

package/dist/log.d.ts

@@ -14,11 +14,6 @@ export declare const initializeLogger: ({ pretty, level }: LoggerOptions) => voi
  * Enable OTEL logging by reconfiguring the logger with multistream.
  * Uses a custom destination that receives full JSON logs (with msg, level, time).
  *
- * The base logger level is set to 'debug' so all logs are generated,
- * while each stream filters to its own level:
- * - Terminal: user-specified level (default: 'info')
- * - OTEL/Cloud: always 'debug' to capture all logs for observability
- *
  * @internal
  */
 export declare const enableOtelLogging: () => void;

package/dist/log.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAqB,MAAM,EAAE,MAAM,MAAM,CAAC;AAKtD,gBAAgB;AAChB,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,gBAAgB;AAChB,eAAO,IAAI,aAAa,EAAE,aAAa,CAAC;AAQxC,gBAAgB;AAChB,eAAO,MAAM,GAAG,cAKf,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,gBAAgB,sBAAuB,aAAa,SAMhE,CAAC;AAqBF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB,YAiB7B,CAAC"}
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAqB,MAAM,EAAE,MAAM,MAAM,CAAC;AAKtD,gBAAgB;AAChB,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,gBAAgB;AAChB,eAAO,IAAI,aAAa,EAAE,aAAa,CAAC;AAQxC,gBAAgB;AAChB,eAAO,MAAM,GAAG,cAKf,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,gBAAgB,sBAAuB,aAAa,SAMhE,CAAC;AAqBF;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,YAgB7B,CAAC"}

package/dist/log.js

@@ -38,12 +38,12 @@ const enableOtelLogging = () => {
   }
   otelEnabled = true;
   const { pretty, level } = loggerOptions;
-  const terminalLevel = level || "info";
+  const logLevel = level || "info";
   const streams = [
-    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: terminalLevel },
+    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: logLevel },
     { stream: new OtelDestination(), level: "debug" }
   ];
-  logger = pino({ level: "debug" }, multistream(streams));
+  logger = pino({ level: logLevel }, multistream(streams));
 };
 export {
   enableOtelLogging,

package/dist/log.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/log.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Writable } from 'node:stream';\nimport type { DestinationStream, Logger } from 'pino';\nimport { multistream, pino } from 'pino';\nimport { build as pinoPretty } from 'pino-pretty';\nimport { type PinoLogObject, emitToOtel } from './telemetry/pino_otel_transport.js';\n\n/** @internal */\nexport type LoggerOptions = {\n  pretty: boolean;\n  level?: string;\n};\n\n/** @internal */\nexport let loggerOptions: LoggerOptions;\n\n/** @internal */\nlet logger: Logger | undefined = undefined;\n\n/** @internal */\nlet otelEnabled = false;\n\n/** @internal */\nexport const log = () => {\n  if (!logger) {\n    throw new TypeError('logger not initialized. did you forget to run initializeLogger()?');\n  }\n  return logger;\n};\n\n/** @internal */\nexport const initializeLogger = ({ pretty, level }: LoggerOptions) => {\n  loggerOptions = { pretty, level };\n  logger = pino(\n    { level: level || 'info' },\n    pretty ? pinoPretty({ colorize: true }) : process.stdout,\n  );\n};\n\n/**\n * Custom Pino destination that parses JSON logs and emits to OTEL.\n * This receives the FULL serialized log including msg, level, time, etc.\n */\nclass OtelDestination extends Writable {\n  _write(chunk: Buffer, _encoding: string, callback: (error?: Error | null) => void): void {\n    try {\n      const line = chunk.toString().trim();\n      if (line) {\n        const logObj = JSON.parse(line) as PinoLogObject;\n        emitToOtel(logObj);\n      }\n    } catch {\n      // Ignore parse errors (e.g., non-JSON lines)\n    }\n    callback();\n  }\n}\n\n/**\n * Enable OTEL logging by reconfiguring the logger with multistream.\n * Uses a custom destination that receives full JSON logs (with msg, level, time).\n *\n * The base logger level is set to 'debug' so all logs are generated,\n * while each stream filters to its own level:\n * - Terminal: user-specified level (default: 'info')\n * - OTEL/Cloud: always 'debug' to capture all logs for observability\n *\n * @internal\n */\nexport const enableOtelLogging = () => {\n  if (otelEnabled || !logger) {\n    console.warn('OTEL logging already enabled or logger not initialized');\n    return;\n  }\n  otelEnabled = true;\n\n  const { pretty, level } = loggerOptions;\n\n  const terminalLevel = level || 'info';\n  const streams: { stream: DestinationStream; level: string }[] = [\n    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: terminalLevel },\n    { stream: new OtelDestination(), level: 'debug' },\n  ];\n\n  // Base level must be 'debug' to generate all logs; each stream filters independently\n  logger = pino({ level: 'debug' }, multistream(streams));\n};\n"],"mappings":"AAGA,SAAS,gBAAgB;AAEzB,SAAS,aAAa,YAAY;AAClC,SAAS,SAAS,kBAAkB;AACpC,SAA6B,kBAAkB;AASxC,IAAI;AAGX,IAAI,SAA6B;AAGjC,IAAI,cAAc;AAGX,MAAM,MAAM,MAAM;AACvB,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,UAAU,mEAAmE;AAAA,EACzF;AACA,SAAO;AACT;AAGO,MAAM,mBAAmB,CAAC,EAAE,QAAQ,MAAM,MAAqB;AACpE,kBAAgB,EAAE,QAAQ,MAAM;AAChC,WAAS;AAAA,IACP,EAAE,OAAO,SAAS,OAAO;AAAA,IACzB,SAAS,WAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ;AAAA,EACpD;AACF;AAMA,MAAM,wBAAwB,SAAS;AAAA,EACrC,OAAO,OAAe,WAAmB,UAAgD;AACvF,QAAI;AACF,YAAM,OAAO,MAAM,SAAS,EAAE,KAAK;AACnC,UAAI,MAAM;AACR,cAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,mBAAW,MAAM;AAAA,MACnB;AAAA,IACF,QAAQ;AAAA,IAER;AACA,aAAS;AAAA,EACX;AACF;AAaO,MAAM,oBAAoB,MAAM;AACrC,MAAI,eAAe,CAAC,QAAQ;AAC1B,YAAQ,KAAK,wDAAwD;AACrE;AAAA,EACF;AACA,gBAAc;AAEd,QAAM,EAAE,QAAQ,MAAM,IAAI;AAE1B,QAAM,gBAAgB,SAAS;AAC/B,QAAM,UAA0D;AAAA,IAC9D,EAAE,QAAQ,SAAS,WAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ,QAAQ,OAAO,cAAc;AAAA,IACzF,EAAE,QAAQ,IAAI,gBAAgB,GAAG,OAAO,QAAQ;AAAA,EAClD;AAGA,WAAS,KAAK,EAAE,OAAO,QAAQ,GAAG,YAAY,OAAO,CAAC;AACxD;","names":[]}
+{"version":3,"sources":["../src/log.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Writable } from 'node:stream';\nimport type { DestinationStream, Logger } from 'pino';\nimport { multistream, pino } from 'pino';\nimport { build as pinoPretty } from 'pino-pretty';\nimport { type PinoLogObject, emitToOtel } from './telemetry/pino_otel_transport.js';\n\n/** @internal */\nexport type LoggerOptions = {\n  pretty: boolean;\n  level?: string;\n};\n\n/** @internal */\nexport let loggerOptions: LoggerOptions;\n\n/** @internal */\nlet logger: Logger | undefined = undefined;\n\n/** @internal */\nlet otelEnabled = false;\n\n/** @internal */\nexport const log = () => {\n  if (!logger) {\n    throw new TypeError('logger not initialized. did you forget to run initializeLogger()?');\n  }\n  return logger;\n};\n\n/** @internal */\nexport const initializeLogger = ({ pretty, level }: LoggerOptions) => {\n  loggerOptions = { pretty, level };\n  logger = pino(\n    { level: level || 'info' },\n    pretty ? pinoPretty({ colorize: true }) : process.stdout,\n  );\n};\n\n/**\n * Custom Pino destination that parses JSON logs and emits to OTEL.\n * This receives the FULL serialized log including msg, level, time, etc.\n */\nclass OtelDestination extends Writable {\n  _write(chunk: Buffer, _encoding: string, callback: (error?: Error | null) => void): void {\n    try {\n      const line = chunk.toString().trim();\n      if (line) {\n        const logObj = JSON.parse(line) as PinoLogObject;\n        emitToOtel(logObj);\n      }\n    } catch {\n      // Ignore parse errors (e.g., non-JSON lines)\n    }\n    callback();\n  }\n}\n\n/**\n * Enable OTEL logging by reconfiguring the logger with multistream.\n * Uses a custom destination that receives full JSON logs (with msg, level, time).\n *\n * @internal\n */\nexport const enableOtelLogging = () => {\n  if (otelEnabled || !logger) {\n    console.warn('OTEL logging already enabled or logger not initialized');\n    return;\n  }\n  otelEnabled = true;\n\n  const { pretty, level } = loggerOptions;\n\n  const logLevel = level || 'info';\n  const streams: { stream: DestinationStream; level: string }[] = [\n    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: logLevel },\n    { stream: new OtelDestination(), level: 'debug' },\n  ];\n\n  logger = pino({ level: logLevel }, multistream(streams));\n};\n"],"mappings":"AAGA,SAAS,gBAAgB;AAEzB,SAAS,aAAa,YAAY;AAClC,SAAS,SAAS,kBAAkB;AACpC,SAA6B,kBAAkB;AASxC,IAAI;AAGX,IAAI,SAA6B;AAGjC,IAAI,cAAc;AAGX,MAAM,MAAM,MAAM;AACvB,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI,UAAU,mEAAmE;AAAA,EACzF;AACA,SAAO;AACT;AAGO,MAAM,mBAAmB,CAAC,EAAE,QAAQ,MAAM,MAAqB;AACpE,kBAAgB,EAAE,QAAQ,MAAM;AAChC,WAAS;AAAA,IACP,EAAE,OAAO,SAAS,OAAO;AAAA,IACzB,SAAS,WAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ;AAAA,EACpD;AACF;AAMA,MAAM,wBAAwB,SAAS;AAAA,EACrC,OAAO,OAAe,WAAmB,UAAgD;AACvF,QAAI;AACF,YAAM,OAAO,MAAM,SAAS,EAAE,KAAK;AACnC,UAAI,MAAM;AACR,cAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,mBAAW,MAAM;AAAA,MACnB;AAAA,IACF,QAAQ;AAAA,IAER;AACA,aAAS;AAAA,EACX;AACF;AAQO,MAAM,oBAAoB,MAAM;AACrC,MAAI,eAAe,CAAC,QAAQ;AAC1B,YAAQ,KAAK,wDAAwD;AACrE;AAAA,EACF;AACA,gBAAc;AAEd,QAAM,EAAE,QAAQ,MAAM,IAAI;AAE1B,QAAM,WAAW,SAAS;AAC1B,QAAM,UAA0D;AAAA,IAC9D,EAAE,QAAQ,SAAS,WAAW,EAAE,UAAU,KAAK,CAAC,IAAI,QAAQ,QAAQ,OAAO,SAAS;AAAA,IACpF,EAAE,QAAQ,IAAI,gBAAgB,GAAG,OAAO,QAAQ;AAAA,EAClD;AAEA,WAAS,KAAK,EAAE,OAAO,SAAS,GAAG,YAAY,OAAO,CAAC;AACzD;","names":[]}

package/dist/stt/stt.cjs

@@ -90,7 +90,7 @@ class SpeechStream {
     this.neededSampleRate = sampleRate;
     this.monitorMetrics();
     this.pumpInput();
-    (0, import_utils.startSoon)(() => this.mainTask().then(() => this.queue.close()));
+    (0, import_utils.startSoon)(() => this.mainTask().finally(() => this.queue.close()));
   }
   async mainTask() {
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {