@livekit/agents 0.0.0-20260120144724

package/dist/llm/fallback_adapter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/llm/fallback_adapter.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { beforeAll, describe, expect, it, vi } from 'vitest';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { initializeLogger } from '../log.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { delay } from '../utils.js';\nimport type { ChatContext } from './chat_context.js';\nimport { FallbackAdapter } from './fallback_adapter.js';\nimport { type ChatChunk, LLM, LLMStream } from './llm.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nclass MockLLMStream extends LLMStream {\n  public myLLM: LLM;\n\n  constructor(\n    llm: LLM,\n    opts: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n    private shouldFail: boolean = false,\n    private failAfterChunks: number = 0,\n  ) {\n    super(llm, opts);\n    this.myLLM = llm;\n  }\n\n  protected async run(): Promise<void> {\n    if (this.shouldFail && this.failAfterChunks === 0) {\n      throw new APIError('Mock LLM failed immediately');\n    }\n\n    const chunk: ChatChunk = {\n      id: 'test-id',\n      delta: { role: 'assistant', content: 'chunk' },\n    };\n\n    for (let i = 0; i < 3; i++) {\n      if (this.shouldFail && i === this.failAfterChunks) {\n        throw new APIError('Mock LLM failed after chunks');\n      }\n      this.queue.put(chunk);\n      await delay(10);\n    }\n  }\n}\n\nclass MockLLM extends LLM {\n  shouldFail: boolean = false;\n  failAfterChunks: number = 0;\n  private _label: string;\n\n  constructor(label: string) {\n    super();\n    this._label = label;\n  }\n\n  label(): string {\n    return this._label;\n  }\n\n  chat(opts: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream {\n    return new MockLLMStream(\n      this,\n      {\n        chatCtx: opts.chatCtx,\n        toolCtx: opts.toolCtx,\n        connOptions: opts.connOptions!,\n      },\n      this.shouldFail,\n      this.failAfterChunks,\n    );\n  }\n}\n\ndescribe('FallbackAdapter', () => {\n  beforeAll(() => {\n    initializeLogger({ pretty: false });\n    // Suppress unhandled rejections from LLMStream background tasks\n    process.on('unhandledRejection', () => {});\n  });\n\n  it('should initialize correctly', () => {\n    const llm1 = new MockLLM('llm1');\n    const adapter = new FallbackAdapter({ llms: [llm1] });\n    expect(adapter.llms).toHaveLength(1);\n    expect(adapter.llms[0]).toBe(llm1);\n  });\n\n  it('should throw if no LLMs provided', () => {\n    expect(() => new FallbackAdapter({ llms: [] })).toThrow();\n  });\n\n  it('should use primary LLM if successful', async () => {\n    const llm1 = new MockLLM('llm1');\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    expect(chunks).toHaveLength(3);\n    // Should verify it used llm1 (we can check logs or spy, but simple success is good first step)\n  });\n\n  it('should fallback to second LLM if first fails immediately', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    expect(chunks).toHaveLength(3);\n    expect(adapter._status[0]!.available).toBe(false);\n    expect(adapter._status[1]!.available).toBe(true);\n  });\n\n  it('should fail if all LLMs fail', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    llm2.shouldFail = true;\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const errorPromise = new Promise<Error>((resolve) => {\n      adapter.on('error', (e) => resolve(e.error));\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    const error = await errorPromise;\n    expect(error).toBeInstanceOf(APIConnectionError);\n  });\n\n  it('should fail if chunks sent and retryOnChunkSent is false', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    llm1.failAfterChunks = 1; // Fail after 1 chunk\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({\n      llms: [llm1, llm2],\n      retryOnChunkSent: false,\n    });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const errorPromise = new Promise<Error>((resolve) => {\n      adapter.on('error', (e) => resolve(e.error));\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    const error = await errorPromise;\n    expect(error).toBeInstanceOf(APIError);\n  });\n\n  it('should fallback if chunks sent and retryOnChunkSent is true', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    llm1.failAfterChunks = 1;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({\n      llms: [llm1, llm2],\n      retryOnChunkSent: true,\n    });\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    const chunks: ChatChunk[] = [];\n    for await (const chunk of stream) {\n      chunks.push(chunk);\n    }\n\n    // 1 chunk from failed llm1 + 3 chunks from llm2\n    expect(chunks).toHaveLength(4);\n  });\n\n  it('should emit availability changed events', async () => {\n    const llm1 = new MockLLM('llm1');\n    llm1.shouldFail = true;\n    const llm2 = new MockLLM('llm2');\n    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });\n\n    const eventSpy = vi.fn();\n    (adapter as any).on('llm_availability_changed', eventSpy);\n\n    const stream = adapter.chat({\n      chatCtx: {} as ChatContext,\n    });\n\n    for await (const _ of stream) {\n      // consume\n    }\n\n    expect(eventSpy).toHaveBeenCalledWith(\n      expect.objectContaining({\n        llm: llm1,\n        available: false,\n      }),\n    );\n  });\n});\n"],"mappings":"AAGA,SAAS,WAAW,UAAU,QAAQ,IAAI,UAAU;AACpD,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,wBAAwB;AAEjC,SAAS,aAAa;AAEtB,SAAS,uBAAuB;AAChC,SAAyB,KAAK,iBAAiB;AAG/C,MAAM,sBAAsB,UAAU;AAAA,EAGpC,YACE,KACA,MAKQ,aAAsB,OACtB,kBAA0B,GAClC;AACA,UAAM,KAAK,IAAI;AAHP;AACA;AAGR,SAAK,QAAQ;AAAA,EACf;AAAA,EAdO;AAAA,EAgBP,MAAgB,MAAqB;AACnC,QAAI,KAAK,cAAc,KAAK,oBAAoB,GAAG;AACjD,YAAM,IAAI,SAAS,6BAA6B;AAAA,IAClD;AAEA,UAAM,QAAmB;AAAA,MACvB,IAAI;AAAA,MACJ,OAAO,EAAE,MAAM,aAAa,SAAS,QAAQ;AAAA,IAC/C;AAEA,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,UAAI,KAAK,cAAc,MAAM,KAAK,iBAAiB;AACjD,cAAM,IAAI,SAAS,8BAA8B;AAAA,MACnD;AACA,WAAK,MAAM,IAAI,KAAK;AACpB,YAAM,MAAM,EAAE;AAAA,IAChB;AAAA,EACF;AACF;AAEA,MAAM,gBAAgB,IAAI;AAAA,EACxB,aAAsB;AAAA,EACtB,kBAA0B;AAAA,EAClB;AAAA,EAER,YAAY,OAAe;AACzB,UAAM;AACN,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,QAAgB;AACd,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,KAAK,MAOS;AACZ,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,QACE,SAAS,KAAK;AAAA,QACd,SAAS,KAAK;AAAA,QACd,aAAa,KAAK;AAAA,MACpB;AAAA,MACA,KAAK;AAAA,MACL,KAAK;AAAA,IACP;AAAA,EACF;AACF;AAEA,SAAS,mBAAmB,MAAM;AAChC,YAAU,MAAM;AACd,qBAAiB,EAAE,QAAQ,MAAM,CAAC;AAElC,YAAQ,GAAG,sBAAsB,MAAM;AAAA,IAAC,CAAC;AAAA,EAC3C,CAAC;AAED,KAAG,+BAA+B,MAAM;AACtC,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AACpD,WAAO,QAAQ,IAAI,EAAE,aAAa,CAAC;AACnC,WAAO,QAAQ,KAAK,CAAC,CAAC,EAAE,KAAK,IAAI;AAAA,EACnC,CAAC;AAED,KAAG,oCAAoC,MAAM;AAC3C,WAAO,MAAM,IAAI,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ;AAAA,EAC1D,CAAC;AAED,KAAG,wCAAwC,YAAY;AACrD,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,MAAM,EAAE,aAAa,CAAC;AAAA,EAE/B,CAAC;AAED,KAAG,4DAA4D,YAAY;AACzE,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,MAAM,EAAE,aAAa,CAAC;AAC7B,WAAO,QAAQ,QAAQ,CAAC,EAAG,SAAS,EAAE,KAAK,KAAK;AAChD,WAAO,QAAQ,QAAQ,CAAC,EAAG,SAAS,EAAE,KAAK,IAAI;AAAA,EACjD,CAAC;AAED,KAAG,gCAAgC,YAAY;AAC7C,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,eAAe,IAAI,QAAe,CAAC,YAAY;AACnD,cAAQ,GAAG,SAAS,CAAC,MAAM,QAAQ,EAAE,KAAK,CAAC;AAAA,IAC7C,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,UAAM,QAAQ,MAAM;AACpB,WAAO,KAAK,EAAE,eAAe,kBAAkB;AAAA,EACjD,CAAC;AAED,KAAG,4DAA4D,YAAY;AACzE,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AACvB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB;AAAA,MAClC,MAAM,CAAC,MAAM,IAAI;AAAA,MACjB,kBAAkB;AAAA,IACpB,CAAC;AAED,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,eAAe,IAAI,QAAe,CAAC,YAAY;AACnD,cAAQ,GAAG,SAAS,CAAC,MAAM,QAAQ,EAAE,KAAK,CAAC;AAAA,IAC7C,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,UAAM,QAAQ,MAAM;AACpB,WAAO,KAAK,EAAE,eAAe,QAAQ;AAAA,EACvC,CAAC;AAED,KAAG,+DAA+D,YAAY;AAC5E,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AACvB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB;AAAA,MAClC,MAAM,CAAC,MAAM,IAAI;AAAA,MACjB,kBAAkB;AAAA,IACpB,CAAC;AAED,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,UAAM,SAAsB,CAAC;AAC7B,qBAAiB,SAAS,QAAQ;AAChC,aAAO,KAAK,KAAK;AAAA,IACnB;AAGA,WAAO,MAAM,EAAE,aAAa,CAAC;AAAA,EAC/B,CAAC;AAED,KAAG,2CAA2C,YAAY;AACxD,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,SAAK,aAAa;AAClB,UAAM,OAAO,IAAI,QAAQ,MAAM;AAC/B,UAAM,UAAU,IAAI,gBAAgB,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;AAE1D,UAAM,WAAW,GAAG,GAAG;AACvB,IAAC,QAAgB,GAAG,4BAA4B,QAAQ;AAExD,UAAM,SAAS,QAAQ,KAAK;AAAA,MAC1B,SAAS,CAAC;AAAA,IACZ,CAAC;AAED,qBAAiB,KAAK,QAAQ;AAAA,IAE9B;AAEA,WAAO,QAAQ,EAAE;AAAA,MACf,OAAO,iBAAiB;AAAA,QACtB,KAAK;AAAA,QACL,WAAW;AAAA,MACb,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH,CAAC;","names":[]}

package/dist/llm/index.cjs

@@ -0,0 +1,79 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var llm_exports = {};
+__export(llm_exports, {
+  AgentHandoffItem: () => import_chat_context.AgentHandoffItem,
+  ChatContext: () => import_chat_context.ChatContext,
+  ChatMessage: () => import_chat_context.ChatMessage,
+  FallbackAdapter: () => import_fallback_adapter.FallbackAdapter,
+  FunctionCall: () => import_chat_context.FunctionCall,
+  FunctionCallOutput: () => import_chat_context.FunctionCallOutput,
+  LLM: () => import_llm.LLM,
+  LLMStream: () => import_llm.LLMStream,
+  RealtimeModel: () => import_realtime.RealtimeModel,
+  RealtimeSession: () => import_realtime.RealtimeSession,
+  RemoteChatContext: () => import_remote_chat_context.RemoteChatContext,
+  ToolError: () => import_tool_context.ToolError,
+  computeChatCtxDiff: () => import_utils.computeChatCtxDiff,
+  createAudioContent: () => import_chat_context.createAudioContent,
+  createImageContent: () => import_chat_context.createImageContent,
+  createToolOptions: () => import_utils.createToolOptions,
+  executeToolCall: () => import_utils.executeToolCall,
+  handoff: () => import_tool_context.handoff,
+  isFunctionTool: () => import_tool_context.isFunctionTool,
+  oaiBuildFunctionInfo: () => import_utils.oaiBuildFunctionInfo,
+  oaiParams: () => import_utils.oaiParams,
+  toJsonSchema: () => import_utils.toJsonSchema,
+  tool: () => import_tool_context.tool
+});
+module.exports = __toCommonJS(llm_exports);
+var import_tool_context = require("./tool_context.cjs");
+var import_chat_context = require("./chat_context.cjs");
+var import_llm = require("./llm.cjs");
+var import_realtime = require("./realtime.cjs");
+var import_remote_chat_context = require("./remote_chat_context.cjs");
+var import_utils = require("./utils.cjs");
+var import_fallback_adapter = require("./fallback_adapter.cjs");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AgentHandoffItem,
+  ChatContext,
+  ChatMessage,
+  FallbackAdapter,
+  FunctionCall,
+  FunctionCallOutput,
+  LLM,
+  LLMStream,
+  RealtimeModel,
+  RealtimeSession,
+  RemoteChatContext,
+  ToolError,
+  computeChatCtxDiff,
+  createAudioContent,
+  createImageContent,
+  createToolOptions,
+  executeToolCall,
+  handoff,
+  isFunctionTool,
+  oaiBuildFunctionInfo,
+  oaiParams,
+  toJsonSchema,
+  tool
+});
+//# sourceMappingURL=index.cjs.map

package/dist/llm/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAaO;AAEP,0BAaO;AAIP,iBAOO;AAEP,sBAWO;AAEP,iCAAkC;AAElC,mBAQO;AAEP,8BAIO;","names":[]}

package/dist/llm/index.d.cts

@@ -0,0 +1,9 @@
+export { handoff, isFunctionTool, tool, ToolError, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
+export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, } from './chat_context.js';
+export type { ProviderFormat } from './provider_format/index.js';
+export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage, type LLMCallbacks, } from './llm.js';
+export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';
+export { RemoteChatContext } from './remote_chat_context.js';
+export { computeChatCtxDiff, createToolOptions, executeToolCall, oaiBuildFunctionInfo, oaiParams, toJsonSchema, type OpenAIFunctionParameters, } from './utils.js';
+export { FallbackAdapter, type AvailabilityChangedEvent, type FallbackAdapterOptions, } from './fallback_adapter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/llm/index.d.ts

@@ -0,0 +1,9 @@
+export { handoff, isFunctionTool, tool, ToolError, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
+export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, } from './chat_context.js';
+export type { ProviderFormat } from './provider_format/index.js';
+export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage, type LLMCallbacks, } from './llm.js';
+export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';
+export { RemoteChatContext } from './remote_chat_context.js';
+export { computeChatCtxDiff, createToolOptions, executeToolCall, oaiBuildFunctionInfo, oaiParams, toJsonSchema, type OpenAIFunctionParameters, } from './utils.js';
+export { FallbackAdapter, type AvailabilityChangedEvent, type FallbackAdapterOptions, } from './fallback_adapter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/llm/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,OAAO,EACP,cAAc,EACd,IAAI,EACJ,SAAS,EACT,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,QAAQ,GACd,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EACL,GAAG,EACH,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,aAAa,EACb,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,GACrC,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,SAAS,EACT,YAAY,EACZ,KAAK,wBAAwB,GAC9B,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,eAAe,EACf,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,uBAAuB,CAAC"}

package/dist/llm/index.js

@@ -0,0 +1,61 @@
+import {
+  handoff,
+  isFunctionTool,
+  tool,
+  ToolError
+} from "./tool_context.js";
+import {
+  AgentHandoffItem,
+  ChatContext,
+  ChatMessage,
+  createAudioContent,
+  createImageContent,
+  FunctionCall,
+  FunctionCallOutput
+} from "./chat_context.js";
+import {
+  LLM,
+  LLMStream
+} from "./llm.js";
+import {
+  RealtimeModel,
+  RealtimeSession
+} from "./realtime.js";
+import { RemoteChatContext } from "./remote_chat_context.js";
+import {
+  computeChatCtxDiff,
+  createToolOptions,
+  executeToolCall,
+  oaiBuildFunctionInfo,
+  oaiParams,
+  toJsonSchema
+} from "./utils.js";
+import {
+  FallbackAdapter
+} from "./fallback_adapter.js";
+export {
+  AgentHandoffItem,
+  ChatContext,
+  ChatMessage,
+  FallbackAdapter,
+  FunctionCall,
+  FunctionCallOutput,
+  LLM,
+  LLMStream,
+  RealtimeModel,
+  RealtimeSession,
+  RemoteChatContext,
+  ToolError,
+  computeChatCtxDiff,
+  createAudioContent,
+  createImageContent,
+  createToolOptions,
+  executeToolCall,
+  handoff,
+  isFunctionTool,
+  oaiBuildFunctionInfo,
+  oaiParams,
+  toJsonSchema,
+  tool
+};
+//# sourceMappingURL=index.js.map

package/dist/llm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OASK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAMK;AAIP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OASK;AAEP,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,OAGK;","names":[]}

package/dist/llm/llm.cjs

@@ -0,0 +1,226 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var llm_exports = {};
+__export(llm_exports, {
+  LLM: () => LLM,
+  LLMStream: () => LLMStream
+});
+module.exports = __toCommonJS(llm_exports);
+var import_node_events = require("node:events");
+var import_exceptions = require("../_exceptions.cjs");
+var import_log = require("../log.cjs");
+var import_telemetry = require("../telemetry/index.cjs");
+var import_types = require("../types.cjs");
+var import_utils = require("../utils.cjs");
+var import_chat_context = require("./chat_context.cjs");
+class LLM extends import_node_events.EventEmitter {
+  constructor() {
+    super();
+  }
+  /**
+   * Get the model name/identifier for this LLM instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model() {
+    return "unknown";
+  }
+  /**
+   * Pre-warm connection to the LLM service
+   */
+  prewarm() {
+  }
+  async aclose() {
+  }
+}
+class LLMStream {
+  output = new import_utils.AsyncIterableQueue();
+  queue = new import_utils.AsyncIterableQueue();
+  closed = false;
+  abortController = new AbortController();
+  _connOptions;
+  logger = (0, import_log.log)();
+  #llm;
+  #chatCtx;
+  #toolCtx;
+  #llmRequestSpan;
+  constructor(llm, {
+    chatCtx,
+    toolCtx,
+    connOptions
+  }) {
+    this.#llm = llm;
+    this.#chatCtx = chatCtx;
+    this.#toolCtx = toolCtx;
+    this._connOptions = connOptions;
+    this.monitorMetrics();
+    this.abortController.signal.addEventListener("abort", () => {
+      this.output.close();
+      this.closed = true;
+    });
+    (0, import_utils.startSoon)(() => this.mainTask().finally(() => this.queue.close()));
+  }
+  _mainTaskImpl = async (span) => {
+    this.#llmRequestSpan = span;
+    span.setAttribute(import_telemetry.traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);
+    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+      try {
+        return await import_telemetry.tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(import_telemetry.traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              (0, import_telemetry.recordException)(attemptSpan, (0, import_utils.toError)(error));
+              throw error;
+            }
+          },
+          { name: "llm_request_run" }
+        );
+      } catch (error) {
+        if (error instanceof import_exceptions.APIError) {
+          const retryInterval = (0, import_types.intervalForRetry)(this._connOptions, i);
+          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+            this.emitError({ error, recoverable: false });
+            throw error;
+          } else if (i === this._connOptions.maxRetry) {
+            this.emitError({ error, recoverable: false });
+            throw new import_exceptions.APIConnectionError({
+              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,
+              options: { retryable: false }
+            });
+          } else {
+            this.emitError({ error, recoverable: true });
+            this.logger.warn(
+              { llm: this.#llm.label(), attempt: i + 1, error },
+              `failed to generate LLM completion, retrying in ${retryInterval}s`
+            );
+          }
+          if (retryInterval > 0) {
+            await (0, import_utils.delay)(retryInterval);
+          }
+        } else {
+          this.emitError({ error: (0, import_utils.toError)(error), recoverable: false });
+          throw error;
+        }
+      }
+    }
+  };
+  mainTask = async () => import_telemetry.tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+    name: "llm_request",
+    endOnExit: false
+  });
+  emitError({ error, recoverable }) {
+    this.#llm.emit("error", {
+      type: "llm_error",
+      timestamp: Date.now(),
+      label: this.#llm.label(),
+      error,
+      recoverable
+    });
+  }
+  async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    let ttft = BigInt(-1);
+    let requestId = "";
+    let usage;
+    let completionStartTime;
+    for await (const ev of this.queue) {
+      if (this.abortController.signal.aborted) {
+        break;
+      }
+      this.output.put(ev);
+      requestId = ev.id;
+      if (ttft === BigInt(-1)) {
+        ttft = process.hrtime.bigint() - startTime;
+        completionStartTime = (/* @__PURE__ */ new Date()).toISOString();
+      }
+      if (ev.usage) {
+        usage = ev.usage;
+      }
+    }
+    this.output.close();
+    const duration = process.hrtime.bigint() - startTime;
+    const durationMs = Math.trunc(Number(duration / BigInt(1e6)));
+    const metrics = {
+      type: "llm_metrics",
+      timestamp: Date.now(),
+      requestId,
+      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1e6))),
+      durationMs,
+      cancelled: this.abortController.signal.aborted,
+      label: this.#llm.label(),
+      completionTokens: (usage == null ? void 0 : usage.completionTokens) || 0,
+      promptTokens: (usage == null ? void 0 : usage.promptTokens) || 0,
+      promptCachedTokens: (usage == null ? void 0 : usage.promptCachedTokens) || 0,
+      totalTokens: (usage == null ? void 0 : usage.totalTokens) || 0,
+      tokensPerSecond: (() => {
+        if (durationMs <= 0) {
+          return 0;
+        }
+        return ((usage == null ? void 0 : usage.completionTokens) || 0) / (durationMs / 1e3);
+      })()
+    };
+    if (this.#llmRequestSpan) {
+      this.#llmRequestSpan.setAttribute(import_telemetry.traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));
+      this.#llmRequestSpan.setAttributes({
+        [import_telemetry.traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,
+        [import_telemetry.traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens
+      });
+      if (completionStartTime) {
+        this.#llmRequestSpan.setAttribute(
+          import_telemetry.traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,
+          completionStartTime
+        );
+      }
+      this.#llmRequestSpan.end();
+    }
+    this.#llm.emit("metrics_collected", metrics);
+  }
+  /** The function context of this stream. */
+  get toolCtx() {
+    return this.#toolCtx;
+  }
+  /** The initial chat context of this stream. */
+  get chatCtx() {
+    return this.#chatCtx;
+  }
+  /** The connection options for this stream. */
+  get connOptions() {
+    return this._connOptions;
+  }
+  next() {
+    return this.output.next();
+  }
+  close() {
+    this.abortController.abort();
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  LLM,
+  LLMStream
+});
+//# sourceMappingURL=llm.cjs.map

package/dist/llm/llm.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n  extra?: Record<string, unknown>;\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAEpB,uBAAoD;AACpD,mBAAyD;AACzD,mBAA8D;AAC9D,0BAAmE;AAoC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,4BAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,wBAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,4BAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,oDAAgB,iBAAa,sBAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,oBAAgB,+BAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,wBAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,4BAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,4BAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,4BAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,4BAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/llm/llm.d.cts

@@ -0,0 +1,94 @@
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { LLMMetrics } from '../metrics/base.js';
+import { type APIConnectOptions } from '../types.js';
+import { AsyncIterableQueue } from '../utils.js';
+import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
+export interface ChoiceDelta {
+    role: ChatRole;
+    content?: string;
+    toolCalls?: FunctionCall[];
+    extra?: Record<string, unknown>;
+}
+export interface CompletionUsage {
+    completionTokens: number;
+    promptTokens: number;
+    promptCachedTokens: number;
+    totalTokens: number;
+}
+export interface ChatChunk {
+    id: string;
+    delta?: ChoiceDelta;
+    usage?: CompletionUsage;
+}
+export interface LLMError {
+    type: 'llm_error';
+    timestamp: number;
+    label: string;
+    error: Error;
+    recoverable: boolean;
+}
+export type LLMCallbacks = {
+    ['metrics_collected']: (metrics: LLMMetrics) => void;
+    ['error']: (error: LLMError) => void;
+};
+declare const LLM_base: new () => TypedEmitter<LLMCallbacks>;
+export declare abstract class LLM extends LLM_base {
+    constructor();
+    abstract label(): string;
+    /**
+     * Get the model name/identifier for this LLM instance.
+     *
+     * @returns The model name if available, "unknown" otherwise.
+     *
+     * @remarks
+     * Plugins should override this property to provide their model information.
+     */
+    get model(): string;
+    /**
+     * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
+     */
+    abstract chat({ chatCtx, toolCtx, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
+        chatCtx: ChatContext;
+        toolCtx?: ToolContext;
+        connOptions?: APIConnectOptions;
+        parallelToolCalls?: boolean;
+        toolChoice?: ToolChoice;
+        extraKwargs?: Record<string, unknown>;
+    }): LLMStream;
+    /**
+     * Pre-warm connection to the LLM service
+     */
+    prewarm(): void;
+    aclose(): Promise<void>;
+}
+export declare abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+    #private;
+    protected output: AsyncIterableQueue<ChatChunk>;
+    protected queue: AsyncIterableQueue<ChatChunk>;
+    protected closed: boolean;
+    protected abortController: AbortController;
+    protected _connOptions: APIConnectOptions;
+    protected logger: import("pino").Logger;
+    constructor(llm: LLM, { chatCtx, toolCtx, connOptions, }: {
+        chatCtx: ChatContext;
+        toolCtx?: ToolContext;
+        connOptions: APIConnectOptions;
+    });
+    private _mainTaskImpl;
+    private mainTask;
+    private emitError;
+    protected monitorMetrics(): Promise<void>;
+    protected abstract run(): Promise<void>;
+    /** The function context of this stream. */
+    get toolCtx(): ToolContext | undefined;
+    /** The initial chat context of this stream. */
+    get chatCtx(): ChatContext;
+    /** The connection options for this stream. */
+    get connOptions(): APIConnectOptions;
+    next(): Promise<IteratorResult<ChatChunk>>;
+    close(): void;
+    [Symbol.asyncIterator](): LLMStream;
+}
+export {};
+//# sourceMappingURL=llm.d.ts.map

package/dist/llm/llm.d.ts

@@ -0,0 +1,94 @@
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { LLMMetrics } from '../metrics/base.js';
+import { type APIConnectOptions } from '../types.js';
+import { AsyncIterableQueue } from '../utils.js';
+import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
+export interface ChoiceDelta {
+    role: ChatRole;
+    content?: string;
+    toolCalls?: FunctionCall[];
+    extra?: Record<string, unknown>;
+}
+export interface CompletionUsage {
+    completionTokens: number;
+    promptTokens: number;
+    promptCachedTokens: number;
+    totalTokens: number;
+}
+export interface ChatChunk {
+    id: string;
+    delta?: ChoiceDelta;
+    usage?: CompletionUsage;
+}
+export interface LLMError {
+    type: 'llm_error';
+    timestamp: number;
+    label: string;
+    error: Error;
+    recoverable: boolean;
+}
+export type LLMCallbacks = {
+    ['metrics_collected']: (metrics: LLMMetrics) => void;
+    ['error']: (error: LLMError) => void;
+};
+declare const LLM_base: new () => TypedEmitter<LLMCallbacks>;
+export declare abstract class LLM extends LLM_base {
+    constructor();
+    abstract label(): string;
+    /**
+     * Get the model name/identifier for this LLM instance.
+     *
+     * @returns The model name if available, "unknown" otherwise.
+     *
+     * @remarks
+     * Plugins should override this property to provide their model information.
+     */
+    get model(): string;
+    /**
+     * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
+     */
+    abstract chat({ chatCtx, toolCtx, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
+        chatCtx: ChatContext;
+        toolCtx?: ToolContext;
+        connOptions?: APIConnectOptions;
+        parallelToolCalls?: boolean;
+        toolChoice?: ToolChoice;
+        extraKwargs?: Record<string, unknown>;
+    }): LLMStream;
+    /**
+     * Pre-warm connection to the LLM service
+     */
+    prewarm(): void;
+    aclose(): Promise<void>;
+}
+export declare abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+    #private;
+    protected output: AsyncIterableQueue<ChatChunk>;
+    protected queue: AsyncIterableQueue<ChatChunk>;
+    protected closed: boolean;
+    protected abortController: AbortController;
+    protected _connOptions: APIConnectOptions;
+    protected logger: import("pino").Logger;
+    constructor(llm: LLM, { chatCtx, toolCtx, connOptions, }: {
+        chatCtx: ChatContext;
+        toolCtx?: ToolContext;
+        connOptions: APIConnectOptions;
+    });
+    private _mainTaskImpl;
+    private mainTask;
+    private emitError;
+    protected monitorMetrics(): Promise<void>;
+    protected abstract run(): Promise<void>;
+    /** The function context of this stream. */
+    get toolCtx(): ToolContext | undefined;
+    /** The initial chat context of this stream. */
+    get chatCtx(): ChatContext;
+    /** The connection options for this stream. */
+    get connOptions(): APIConnectOptions;
+    next(): Promise<IteratorResult<ChatChunk>>;
+    close(): void;
+    [Symbol.asyncIterator](): LLMStream;
+}
+export {};
+//# sourceMappingURL=llm.d.ts.map

package/dist/llm/llm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAKhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAAoB,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACvF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAKtF,QAAQ,CAAC,KAAK,IAAI,MAAM;IAExB;;;;;;;OAOG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,UAAU,EACV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACvC,GAAG,SAAS;IAEb;;OAEG;IACH,OAAO,IAAI,IAAI;IAIT,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,MAAM,gCAAuC;IACvD,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,eAAe,kBAAyB;IAClD,SAAS,CAAC,YAAY,EAAE,iBAAiB,CAAC;IAC1C,SAAS,CAAC,MAAM,wBAAS;gBAQvB,GAAG,EAAE,GAAG,EACR,EACE,OAAO,EACP,OAAO,EACP,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,EAAE,iBAAiB,CAAC;KAChC;IAoBH,OAAO,CAAC,aAAa,CAgDnB;IAEF,OAAO,CAAC,QAAQ,CAIX;IAEL,OAAO,CAAC,SAAS;cAUD,cAAc;IAmE9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,2CAA2C;IAC3C,IAAI,OAAO,IAAI,WAAW,GAAG,SAAS,CAErC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,8CAA8C;IAC9C,IAAI,WAAW,IAAI,iBAAiB,CAEnC;IAED,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}