@livekit/agents 1.0.48 → 1.1.0-dev.0

package/dist/llm/index.d.cts

@@ -1,5 +1,5 @@
 export { handoff, isFunctionTool, tool, ToolError, ToolFlag, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
-export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, } from './chat_context.js';
+export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, type MetricsReport, } from './chat_context.js';
 export type { ProviderFormat } from './provider_format/index.js';
 export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage, type LLMCallbacks, } from './llm.js';
 export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';

package/dist/llm/index.d.ts

@@ -1,5 +1,5 @@
 export { handoff, isFunctionTool, tool, ToolError, ToolFlag, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, type ToolContext, type ToolOptions, type ToolType, } from './tool_context.js';
-export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, } from './chat_context.js';
+export { AgentHandoffItem, ChatContext, ChatMessage, createAudioContent, createImageContent, FunctionCall, FunctionCallOutput, type AudioContent, type ChatContent, type ChatItem, type ChatRole, type ImageContent, type MetricsReport, } from './chat_context.js';
 export type { ProviderFormat } from './provider_format/index.js';
 export { LLM, LLMStream, type ChatChunk, type ChoiceDelta, type CompletionUsage, type LLMCallbacks, } from './llm.js';
 export { RealtimeModel, RealtimeSession, type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSpeechStoppedEvent, type InputTranscriptionCompleted, type MessageGeneration, type RealtimeCapabilities, type RealtimeModelError, type RealtimeSessionReconnectedEvent, } from './realtime.js';

package/dist/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,OAAO,EACP,cAAc,EACd,IAAI,EACJ,SAAS,EACT,QAAQ,EACR,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,QAAQ,GACd,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EACL,GAAG,EACH,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,aAAa,EACb,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,GACrC,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,SAAS,EACT,YAAY,EACZ,KAAK,wBAAwB,GAC9B,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,eAAe,EACf,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,OAAO,EACP,cAAc,EACd,IAAI,EACJ,SAAS,EACT,QAAQ,EACR,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,QAAQ,GACd,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,QAAQ,EACb,KAAK,YAAY,EACjB,KAAK,aAAa,GACnB,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EACL,GAAG,EACH,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,aAAa,EACb,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,GACrC,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,SAAS,EACT,YAAY,EACZ,KAAK,wBAAwB,GAC9B,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,eAAe,EACf,KAAK,wBAAwB,EAC7B,KAAK,sBAAsB,GAC5B,MAAM,uBAAuB,CAAC"}

package/dist/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  ToolFlag,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OASK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAMK;AAIP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OASK;AAEP,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,OAGK;","names":[]}
+{"version":3,"sources":["../../src/llm/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport {\n  handoff,\n  isFunctionTool,\n  tool,\n  ToolError,\n  ToolFlag,\n  type AgentHandoff,\n  type FunctionTool,\n  type ProviderDefinedTool,\n  type Tool,\n  type ToolChoice,\n  type ToolContext,\n  type ToolOptions,\n  type ToolType,\n} from './tool_context.js';\n\nexport {\n  AgentHandoffItem,\n  ChatContext,\n  ChatMessage,\n  createAudioContent,\n  createImageContent,\n  FunctionCall,\n  FunctionCallOutput,\n  type AudioContent,\n  type ChatContent,\n  type ChatItem,\n  type ChatRole,\n  type ImageContent,\n  type MetricsReport,\n} from './chat_context.js';\n\nexport type { ProviderFormat } from './provider_format/index.js';\n\nexport {\n  LLM,\n  LLMStream,\n  type ChatChunk,\n  type ChoiceDelta,\n  type CompletionUsage,\n  type LLMCallbacks,\n} from './llm.js';\n\nexport {\n  RealtimeModel,\n  RealtimeSession,\n  type GenerationCreatedEvent,\n  type InputSpeechStartedEvent,\n  type InputSpeechStoppedEvent,\n  type InputTranscriptionCompleted,\n  type MessageGeneration,\n  type RealtimeCapabilities,\n  type RealtimeModelError,\n  type RealtimeSessionReconnectedEvent,\n} from './realtime.js';\n\nexport { RemoteChatContext } from './remote_chat_context.js';\n\nexport {\n  computeChatCtxDiff,\n  createToolOptions,\n  executeToolCall,\n  oaiBuildFunctionInfo,\n  oaiParams,\n  toJsonSchema,\n  type OpenAIFunctionParameters,\n} from './utils.js';\n\nexport {\n  FallbackAdapter,\n  type AvailabilityChangedEvent,\n  type FallbackAdapterOptions,\n} from './fallback_adapter.js';\n"],"mappings":"AAGA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OASK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAOK;AAIP;AAAA,EACE;AAAA,EACA;AAAA,OAKK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OASK;AAEP,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,OAGK;","names":[]}

package/dist/llm/llm.cjs

@@ -44,6 +44,17 @@ class LLM extends import_node_events.EventEmitter {
   get model() {
     return "unknown";
   }
+  /**
+   * Get the provider name for this LLM instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider() {
+    return "unknown";
+  }
   /**
    * Pre-warm connection to the LLM service
    */
@@ -178,7 +189,11 @@ class LLMStream {
           return 0;
         }
         return ((usage == null ? void 0 : usage.completionTokens) || 0) / (durationMs / 1e3);
-      })()
+      })(),
+      metadata: {
+        modelProvider: this.#llm.provider,
+        modelName: this.#llm.model
+      }
     };
     if (this.#llmRequestSpan) {
       this.#llmRequestSpan.setAttribute(import_telemetry.traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));

package/dist/llm/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n  extra?: Record<string, unknown>;\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}ms`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAEpB,uBAAoD;AACpD,mBAAyD;AACzD,mBAA8D;AAC9D,0BAAmE;AAoC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,4BAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,wBAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,4BAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,oDAAgB,iBAAa,sBAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,oBAAgB,+BAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,wBAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,4BAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,4BAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,4BAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,4BAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n  extra?: Record<string, unknown>;\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Get the provider name for this LLM instance.\n   *\n   * @returns The provider name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their provider information.\n   */\n  get provider(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}ms`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n      metadata: {\n        modelProvider: this.#llm.provider,\n        modelName: this.#llm.model,\n      },\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAC7B,wBAA6C;AAC7C,iBAAoB;AAEpB,uBAAoD;AACpD,mBAAyD;AACzD,mBAA8D;AAC9D,0BAAmE;AAoC5D,MAAe,YAAa,gCAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,aAAS,gBAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,gCAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,4BAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,wBAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,4BAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,oDAAgB,iBAAa,sBAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,4BAAU;AAC7B,gBAAM,oBAAgB,+BAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,qCAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,sBAAM,oBAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,WAAO,sBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,wBAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,MACH,UAAU;AAAA,QACR,eAAe,KAAK,KAAK;AAAA,QACzB,WAAW,KAAK,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,4BAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,4BAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,4BAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,4BAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/llm/llm.d.cts

@@ -45,6 +45,15 @@ export declare abstract class LLM extends LLM_base {
      * Plugins should override this property to provide their model information.
      */
     get model(): string;
+    /**
+     * Get the provider name for this LLM instance.
+     *
+     * @returns The provider name if available, "unknown" otherwise.
+     *
+     * @remarks
+     * Plugins should override this property to provide their provider information.
+     */
+    get provider(): string;
     /**
      * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
      */

package/dist/llm/llm.d.ts

@@ -45,6 +45,15 @@ export declare abstract class LLM extends LLM_base {
      * Plugins should override this property to provide their model information.
      */
     get model(): string;
+    /**
+     * Get the provider name for this LLM instance.
+     *
+     * @returns The provider name if available, "unknown" otherwise.
+     *
+     * @remarks
+     * Plugins should override this property to provide their provider information.
+     */
+    get provider(): string;
     /**
      * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
      */

package/dist/llm/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAKhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAAoB,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACvF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAKtF,QAAQ,CAAC,KAAK,IAAI,MAAM;IAExB;;;;;;;OAOG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,UAAU,EACV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACvC,GAAG,SAAS;IAEb;;OAEG;IACH,OAAO,IAAI,IAAI;IAIT,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,MAAM,gCAAuC;IACvD,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,eAAe,kBAAyB;IAClD,SAAS,CAAC,YAAY,EAAE,iBAAiB,CAAC;IAC1C,SAAS,CAAC,MAAM,wBAAS;gBAQvB,GAAG,EAAE,GAAG,EACR,EACE,OAAO,EACP,OAAO,EACP,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,EAAE,iBAAiB,CAAC;KAChC;IAoBH,OAAO,CAAC,aAAa,CAgDnB;IAEF,OAAO,CAAC,QAAQ,CAIX;IAEL,OAAO,CAAC,SAAS;cAUD,cAAc;IAmE9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,2CAA2C;IAC3C,IAAI,OAAO,IAAI,WAAW,GAAG,SAAS,CAErC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,8CAA8C;IAC9C,IAAI,WAAW,IAAI,iBAAiB,CAEnC;IAED,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAKhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAAoB,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACvF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAKtF,QAAQ,CAAC,KAAK,IAAI,MAAM;IAExB;;;;;;;OAOG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;;;;;;OAOG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,UAAU,EACV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACvC,GAAG,SAAS;IAEb;;OAEG;IACH,OAAO,IAAI,IAAI;IAIT,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,MAAM,gCAAuC;IACvD,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,eAAe,kBAAyB;IAClD,SAAS,CAAC,YAAY,EAAE,iBAAiB,CAAC;IAC1C,SAAS,CAAC,MAAM,wBAAS;gBAQvB,GAAG,EAAE,GAAG,EACR,EACE,OAAO,EACP,OAAO,EACP,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,EAAE,iBAAiB,CAAC;KAChC;IAoBH,OAAO,CAAC,aAAa,CAgDnB;IAEF,OAAO,CAAC,QAAQ,CAIX;IAEL,OAAO,CAAC,SAAS;cAUD,cAAc;IAuE9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,2CAA2C;IAC3C,IAAI,OAAO,IAAI,WAAW,GAAG,SAAS,CAErC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,8CAA8C;IAC9C,IAAI,WAAW,IAAI,iBAAiB,CAEnC;IAED,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}

package/dist/llm/llm.js

@@ -20,6 +20,17 @@ class LLM extends EventEmitter {
   get model() {
     return "unknown";
   }
+  /**
+   * Get the provider name for this LLM instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider() {
+    return "unknown";
+  }
   /**
    * Pre-warm connection to the LLM service
    */
@@ -154,7 +165,11 @@ class LLMStream {
           return 0;
         }
         return ((usage == null ? void 0 : usage.completionTokens) || 0) / (durationMs / 1e3);
-      })()
+      })(),
+      metadata: {
+        modelProvider: this.#llm.provider,
+        modelName: this.#llm.model
+      }
     };
     if (this.#llmRequestSpan) {
       this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));

package/dist/llm/llm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n  extra?: Record<string, unknown>;\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}ms`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAC7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,WAAW;AAEpB,SAAS,iBAAiB,YAAY,cAAc;AACpD,SAAiC,wBAAwB;AACzD,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAC9D,eAAmE;AAoC5D,MAAe,YAAa,aAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,SAAS,IAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,WAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,OAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,WAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,8BAAgB,aAAa,QAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,iBAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,OAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,WAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,WAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,WAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,WAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport type { Span } from '@opentelemetry/api';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { recordException, traceTypes, tracer } from '../telemetry/index.js';\nimport { type APIConnectOptions, intervalForRetry } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n  extra?: Record<string, unknown>;\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Get the provider name for this LLM instance.\n   *\n   * @returns The provider name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their provider information.\n   */\n  get provider(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n  #llmRequestSpan?: Span;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().finally(() => this.queue.close()));\n  }\n\n  private _mainTaskImpl = async (span: Span) => {\n    this.#llmRequestSpan = span;\n    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);\n\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await tracer.startActiveSpan(\n          async (attemptSpan) => {\n            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);\n            try {\n              return await this.run();\n            } catch (error) {\n              recordException(attemptSpan, toError(error));\n              throw error;\n            }\n          },\n          { name: 'llm_request_run' },\n        );\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = intervalForRetry(this._connOptions, i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}ms`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  };\n\n  private mainTask = async () =>\n    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {\n      name: 'llm_request',\n      endOnExit: false,\n    });\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n    let completionStartTime: string | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n        completionStartTime = new Date().toISOString();\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n      metadata: {\n        modelProvider: this.#llm.provider,\n        modelName: this.#llm.model,\n      },\n    };\n\n    if (this.#llmRequestSpan) {\n      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));\n\n      this.#llmRequestSpan.setAttributes({\n        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,\n        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,\n      });\n\n      if (completionStartTime) {\n        this.#llmRequestSpan.setAttribute(\n          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,\n          completionStartTime,\n        );\n      }\n\n      // End the span now that metrics are collected\n      this.#llmRequestSpan.end();\n    }\n\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAC7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,WAAW;AAEpB,SAAS,iBAAiB,YAAY,cAAc;AACpD,SAAiC,wBAAwB;AACzD,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAC9D,eAAmE;AAoC5D,MAAe,YAAa,aAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,SAAS,IAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,QAAQ,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAO,SAAe;AAC5C,SAAK,kBAAkB;AACvB,SAAK,aAAa,WAAW,2BAA2B,KAAK,KAAK,KAAK;AAEvE,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,OAAO;AAAA,UAClB,OAAO,gBAAgB;AACrB,wBAAY,aAAa,WAAW,kBAAkB,CAAC;AACvD,gBAAI;AACF,qBAAO,MAAM,KAAK,IAAI;AAAA,YACxB,SAAS,OAAO;AACd,8BAAgB,aAAa,QAAQ,KAAK,CAAC;AAC3C,oBAAM;AAAA,YACR;AAAA,UACF;AAAA,UACA,EAAE,MAAM,kBAAkB;AAAA,QAC5B;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,iBAAiB,KAAK,cAAc,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,WAAW,YACjB,OAAO,gBAAgB,OAAO,SAAS,KAAK,cAAc,IAAI,GAAG;AAAA,IAC/D,MAAM;AAAA,IACN,WAAW;AAAA,EACb,CAAC;AAAA,EAEK,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AACJ,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AACjC,+BAAsB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC/C;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,MACH,UAAU;AAAA,QACR,eAAe,KAAK,KAAK;AAAA,QACzB,WAAW,KAAK,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,QAAI,KAAK,iBAAiB;AACxB,WAAK,gBAAgB,aAAa,WAAW,kBAAkB,KAAK,UAAU,OAAO,CAAC;AAEtF,WAAK,gBAAgB,cAAc;AAAA,QACjC,CAAC,WAAW,8BAA8B,GAAG,QAAQ;AAAA,QACrD,CAAC,WAAW,+BAA+B,GAAG,QAAQ;AAAA,MACxD,CAAC;AAED,UAAI,qBAAqB;AACvB,aAAK,gBAAgB;AAAA,UACnB,WAAW;AAAA,UACX;AAAA,QACF;AAAA,MACF;AAGA,WAAK,gBAAgB,IAAI;AAAA,IAC3B;AAEA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/llm/realtime.cjs

@@ -33,6 +33,9 @@ class RealtimeModel {
   get capabilities() {
     return this._capabilities;
   }
+  get provider() {
+    return "unknown";
+  }
 }
 class RealtimeSession extends import_events.EventEmitter {
   _realtimeModel;

package/dist/llm/realtime.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { TimedString } from '../voice/io.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  /**\n   * Text stream that may contain plain strings or TimedString objects with timestamps.\n   */\n  textStream: ReadableStream<string | TimedString>;\n  audioStream: ReadableStream<AudioFrame>;\n  modalities?: Promise<('text' | 'audio')[]>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n  /** Response ID for correlating metrics with spans */\n  responseId?: string;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n  audioOutput: boolean;\n  manualFunctionCalls: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  /** The model name/identifier used by this realtime model */\n  abstract get model(): string;\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: {\n    messageId: string;\n    audioEndMs: number;\n    modalities?: ('text' | 'audio')[];\n    audioTranscript?: string;\n  }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAA6B;AAE7B,6BAAuC;AACvC,mBAAqB;AAsDd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAQF;AAEO,MAAe,wBAAwB,2BAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,8CAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAiDA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { TimedString } from '../voice/io.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  /**\n   * Text stream that may contain plain strings or TimedString objects with timestamps.\n   */\n  textStream: ReadableStream<string | TimedString>;\n  audioStream: ReadableStream<AudioFrame>;\n  modalities?: Promise<('text' | 'audio')[]>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n  /** Response ID for correlating metrics with spans */\n  responseId?: string;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n  audioOutput: boolean;\n  manualFunctionCalls: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  /** The model name/identifier used by this realtime model */\n  abstract get model(): string;\n\n  get provider(): string {\n    return 'unknown';\n  }\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: {\n    messageId: string;\n    audioEndMs: number;\n    modalities?: ('text' | 'audio')[];\n    audioTranscript?: string;\n  }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAA6B;AAE7B,6BAAuC;AACvC,mBAAqB;AAsDd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAAA,EAKA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAKF;AAEO,MAAe,wBAAwB,2BAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,8CAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAiDA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}

package/dist/llm/realtime.d.cts

@@ -54,6 +54,7 @@ export declare abstract class RealtimeModel {
     get capabilities(): RealtimeCapabilities;
     /** The model name/identifier used by this realtime model */
     abstract get model(): string;
+    get provider(): string;
     abstract session(): RealtimeSession;
     abstract close(): Promise<void>;
 }

package/dist/llm/realtime.d.ts

@@ -54,6 +54,7 @@ export declare abstract class RealtimeModel {
     get capabilities(): RealtimeCapabilities;
     /** The model name/identifier used by this realtime model */
     abstract get model(): string;
+    get provider(): string;
     abstract session(): RealtimeSession;
     abstract close(): Promise<void>;
 }

package/dist/llm/realtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"realtime.d.ts","sourceRoot":"","sources":["../../src/llm/realtime.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACnE,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAE7C,MAAM,WAAW,uBAAuB;IACtC,wBAAwB,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,UAAU,EAAE,cAAc,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;IACjD,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;CAC5C;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,cAAc,CAAC,iBAAiB,CAAC,CAAC;IACjD,cAAc,EAAE,cAAc,CAAC,YAAY,CAAC,CAAC;IAC7C,aAAa,EAAE,OAAO,CAAC;IACvB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,uBAAuB,EAAE,OAAO,CAAC;IACjC,WAAW,EAAE,OAAO,CAAC;IACrB,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,+BAA+B;CAAG;AAEnD,8BAAsB,aAAa;IACjC,OAAO,CAAC,aAAa,CAAuB;gBAEhC,YAAY,EAAE,oBAAoB;IAI9C,IAAI,YAAY,yBAEf;IAED,4DAA4D;IAC5D,QAAQ,KAAK,KAAK,IAAI,MAAM,CAAC;IAE7B,QAAQ,CAAC,OAAO,IAAI,eAAe;IAEnC,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAChC;AAED,8BAAsB,eAAgB,SAAQ,YAAY;IACxD,SAAS,CAAC,cAAc,EAAE,aAAa,CAAC;IACxC,OAAO,CAAC,mBAAmB,CAA4C;IACvE,OAAO,CAAC,SAAS,CAAa;gBAElB,aAAa,EAAE,aAAa;IAMxC,IAAI,aAAa,kBAEhB;IAED,QAAQ,KAAK,OAAO,IAAI,WAAW,CAAC;IAEpC,QAAQ,KAAK,KAAK,IAAI,WAAW,CAAC;IAElC,QAAQ,CAAC,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAEhE;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3D,QAAQ,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEvD,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAEzE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAE3C;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAE9E;;OAEG;IACH,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC;QAClC,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC,IAAI,CAAC;IAEX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;OAEG;IACH,iBAAiB,IAAI,IAAI;YAIX,aAAa;IAW3B,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;CAGnE"}
+{"version":3,"file":"realtime.d.ts","sourceRoot":"","sources":["../../src/llm/realtime.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACnE,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAE7C,MAAM,WAAW,uBAAuB;IACtC,wBAAwB,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,UAAU,EAAE,cAAc,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;IACjD,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;CAC5C;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,cAAc,CAAC,iBAAiB,CAAC,CAAC;IACjD,cAAc,EAAE,cAAc,CAAC,YAAY,CAAC,CAAC;IAC7C,aAAa,EAAE,OAAO,CAAC;IACvB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,uBAAuB,EAAE,OAAO,CAAC;IACjC,WAAW,EAAE,OAAO,CAAC;IACrB,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,+BAA+B;CAAG;AAEnD,8BAAsB,aAAa;IACjC,OAAO,CAAC,aAAa,CAAuB;gBAEhC,YAAY,EAAE,oBAAoB;IAI9C,IAAI,YAAY,yBAEf;IAED,4DAA4D;IAC5D,QAAQ,KAAK,KAAK,IAAI,MAAM,CAAC;IAE7B,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,QAAQ,CAAC,OAAO,IAAI,eAAe;IAEnC,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAChC;AAED,8BAAsB,eAAgB,SAAQ,YAAY;IACxD,SAAS,CAAC,cAAc,EAAE,aAAa,CAAC;IACxC,OAAO,CAAC,mBAAmB,CAA4C;IACvE,OAAO,CAAC,SAAS,CAAa;gBAElB,aAAa,EAAE,aAAa;IAMxC,IAAI,aAAa,kBAEhB;IAED,QAAQ,KAAK,OAAO,IAAI,WAAW,CAAC;IAEpC,QAAQ,KAAK,KAAK,IAAI,WAAW,CAAC;IAElC,QAAQ,CAAC,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAEhE;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3D,QAAQ,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEvD,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAEzE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAE3C;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAE9E;;OAEG;IACH,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC;QAClC,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC,IAAI,CAAC;IAEX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;OAEG;IACH,iBAAiB,IAAI,IAAI;YAIX,aAAa;IAW3B,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;CAGnE"}

package/dist/llm/realtime.js

@@ -9,6 +9,9 @@ class RealtimeModel {
   get capabilities() {
     return this._capabilities;
   }
+  get provider() {
+    return "unknown";
+  }
 }
 class RealtimeSession extends EventEmitter {
   _realtimeModel;

package/dist/llm/realtime.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { TimedString } from '../voice/io.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  /**\n   * Text stream that may contain plain strings or TimedString objects with timestamps.\n   */\n  textStream: ReadableStream<string | TimedString>;\n  audioStream: ReadableStream<AudioFrame>;\n  modalities?: Promise<('text' | 'audio')[]>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n  /** Response ID for correlating metrics with spans */\n  responseId?: string;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n  audioOutput: boolean;\n  manualFunctionCalls: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  /** The model name/identifier used by this realtime model */\n  abstract get model(): string;\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: {\n    messageId: string;\n    audioEndMs: number;\n    modalities?: ('text' | 'audio')[];\n    audioTranscript?: string;\n  }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,8BAA8B;AACvC,SAAS,YAAY;AAsDd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAQF;AAEO,MAAe,wBAAwB,aAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,uBAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAiDA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { TimedString } from '../voice/io.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  /**\n   * Text stream that may contain plain strings or TimedString objects with timestamps.\n   */\n  textStream: ReadableStream<string | TimedString>;\n  audioStream: ReadableStream<AudioFrame>;\n  modalities?: Promise<('text' | 'audio')[]>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n  /** Response ID for correlating metrics with spans */\n  responseId?: string;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n  audioOutput: boolean;\n  manualFunctionCalls: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  /** The model name/identifier used by this realtime model */\n  abstract get model(): string;\n\n  get provider(): string {\n    return 'unknown';\n  }\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: {\n    messageId: string;\n    audioEndMs: number;\n    modalities?: ('text' | 'audio')[];\n    audioTranscript?: string;\n  }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,8BAA8B;AACvC,SAAS,YAAY;AAsDd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAAA,EAKA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAKF;AAEO,MAAe,wBAAwB,aAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,uBAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAiDA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}

package/dist/metrics/base.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/metrics/base.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport type AgentMetrics =\n  | STTMetrics\n  | LLMMetrics\n  | TTSMetrics\n  | VADMetrics\n  | EOUMetrics\n  | RealtimeModelMetrics;\n\nexport type LLMMetrics = {\n  type: 'llm_metrics';\n  label: string;\n  requestId: string;\n  timestamp: number;\n  /** Duration of the request in milliseconds. */\n  durationMs: number;\n  /** Time to first token in milliseconds. */\n  ttftMs: number;\n  cancelled: boolean;\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n  tokensPerSecond: number;\n  speechId?: string;\n};\n\nexport type STTMetrics = {\n  type: 'stt_metrics';\n  label: string;\n  requestId: string;\n  timestamp: number;\n  /**\n   * The request duration in milliseconds, 0.0 if the STT is streaming.\n   */\n  durationMs: number;\n  /**\n   * The duration of the pushed audio in milliseconds.\n   */\n  audioDurationMs: number;\n  /**\n   * Whether the STT is streaming (e.g using websocket).\n   */\n  streamed: boolean;\n};\n\nexport type TTSMetrics = {\n  type: 'tts_metrics';\n  label: string;\n  requestId: string;\n  timestamp: number;\n  /** Time to first byte in milliseconds. */\n  ttfbMs: number;\n  /** Total synthesis duration in milliseconds. */\n  durationMs: number;\n  /** Generated audio duration in milliseconds. */\n  audioDurationMs: number;\n  cancelled: boolean;\n  charactersCount: number;\n  streamed: boolean;\n  segmentId?: string;\n  speechId?: string;\n};\n\nexport type VADMetrics = {\n  type: 'vad_metrics';\n  label: string;\n  timestamp: number;\n  idleTimeMs: number;\n  inferenceDurationTotalMs: number;\n  inferenceCount: number;\n};\n\nexport type EOUMetrics = {\n  type: 'eou_metrics';\n  timestamp: number;\n  /**\n   * Amount of time between the end of speech from VAD and the decision to end the user's turn.\n   * Set to 0.0 if the end of speech was not detected.\n   */\n  endOfUtteranceDelayMs: number;\n  /**\n   * Time taken to obtain the transcript after the end of the user's speech.\n   * Set to 0.0 if the end of speech was not detected.\n   */\n  transcriptionDelayMs: number;\n  /**\n   * Time taken to invoke the user's `Agent.onUserTurnCompleted` callback.\n   */\n  onUserTurnCompletedDelayMs: number;\n  /**\n   * The time the user stopped speaking.\n   */\n  lastSpeakingTimeMs: number;\n  /**\n   * The ID of the speech handle.\n   */\n  speechId?: string;\n};\n\nexport type RealtimeModelMetricsCachedTokenDetails = {\n  audioTokens: number;\n  textTokens: number;\n  imageTokens: number;\n};\n\nexport type RealtimeModelMetricsInputTokenDetails = {\n  audioTokens: number;\n  textTokens: number;\n  imageTokens: number;\n  cachedTokens: number;\n  cachedTokensDetails?: RealtimeModelMetricsCachedTokenDetails;\n};\n\nexport type RealtimeModelMetricsOutputTokenDetails = {\n  textTokens: number;\n  audioTokens: number;\n  imageTokens: number;\n};\n\nexport type RealtimeModelMetrics = {\n  type: 'realtime_model_metrics';\n  label: string;\n  requestId: string;\n  /**\n   * The timestamp of the response creation.\n   */\n  timestamp: number;\n  /**\n   * The duration of the response from created to done in milliseconds.\n   */\n  durationMs: number;\n  /**\n   * Time to first audio token in milliseconds. -1 if no audio token was sent.\n   */\n  ttftMs: number;\n  /**\n   * Whether the request was cancelled.\n   */\n  cancelled: boolean;\n  /**\n   * The number of input tokens used in the Response, including text and audio tokens.\n   */\n  inputTokens: number;\n  /**\n   * The number of output tokens sent in the Response, including text and audio tokens.\n   */\n  outputTokens: number;\n  /**\n   * The total number of tokens in the Response.\n   */\n  totalTokens: number;\n  /**\n   * The number of tokens per second.\n   */\n  tokensPerSecond: number;\n  /**\n   * Details about the input tokens used in the Response.\n   */\n  inputTokenDetails: RealtimeModelMetricsInputTokenDetails;\n  /**\n   * Details about the output tokens used in the Response.\n   */\n  outputTokenDetails: RealtimeModelMetricsOutputTokenDetails;\n};\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]}
+{"version":3,"sources":["../../src/metrics/base.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport type MetricsMetadata = {\n  /** The provider name (e.g., 'openai', 'anthropic'). */\n  modelProvider?: string;\n  /** The model name (e.g., 'gpt-4o', 'claude-3-5-sonnet'). */\n  modelName?: string;\n};\n\nexport type AgentMetrics =\n  | STTMetrics\n  | LLMMetrics\n  | TTSMetrics\n  | VADMetrics\n  | EOUMetrics\n  | RealtimeModelMetrics\n  | InterruptionMetrics;\n\nexport type LLMMetrics = {\n  type: 'llm_metrics';\n  label: string;\n  requestId: string;\n  timestamp: number;\n  /** Duration of the request in milliseconds. */\n  durationMs: number;\n  /** Time to first token in milliseconds. */\n  ttftMs: number;\n  cancelled: boolean;\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n  tokensPerSecond: number;\n  speechId?: string;\n  /** Metadata for model provider and name tracking. */\n  metadata?: MetricsMetadata;\n};\n\nexport type STTMetrics = {\n  type: 'stt_metrics';\n  label: string;\n  requestId: string;\n  timestamp: number;\n  /**\n   * The request duration in milliseconds, 0.0 if the STT is streaming.\n   */\n  durationMs: number;\n  /**\n   * The duration of the pushed audio in milliseconds.\n   */\n  audioDurationMs: number;\n  /** Input audio tokens (for token-based billing). */\n  inputTokens?: number;\n  /** Output text tokens (for token-based billing). */\n  outputTokens?: number;\n  /**\n   * Whether the STT is streaming (e.g using websocket).\n   */\n  streamed: boolean;\n  /** Metadata for model provider and name tracking. */\n  metadata?: MetricsMetadata;\n};\n\nexport type TTSMetrics = {\n  type: 'tts_metrics';\n  label: string;\n  requestId: string;\n  timestamp: number;\n  /** Time to first byte in milliseconds. */\n  ttfbMs: number;\n  /** Total synthesis duration in milliseconds. */\n  durationMs: number;\n  /** Generated audio duration in milliseconds. */\n  audioDurationMs: number;\n  cancelled: boolean;\n  /** Number of characters synthesized (for character-based billing). */\n  charactersCount: number;\n  /** Input text tokens (for token-based billing, e.g., OpenAI TTS). */\n  inputTokens?: number;\n  /** Output audio tokens (for token-based billing, e.g., OpenAI TTS). */\n  outputTokens?: number;\n  streamed: boolean;\n  segmentId?: string;\n  speechId?: string;\n  /** Metadata for model provider and name tracking. */\n  metadata?: MetricsMetadata;\n};\n\nexport type VADMetrics = {\n  type: 'vad_metrics';\n  label: string;\n  timestamp: number;\n  idleTimeMs: number;\n  inferenceDurationTotalMs: number;\n  inferenceCount: number;\n};\n\nexport type EOUMetrics = {\n  type: 'eou_metrics';\n  timestamp: number;\n  /**\n   * Amount of time between the end of speech from VAD and the decision to end the user's turn.\n   * Set to 0.0 if the end of speech was not detected.\n   */\n  endOfUtteranceDelayMs: number;\n  /**\n   * Time taken to obtain the transcript after the end of the user's speech.\n   * Set to 0.0 if the end of speech was not detected.\n   */\n  transcriptionDelayMs: number;\n  /**\n   * Time taken to invoke the user's `Agent.onUserTurnCompleted` callback.\n   */\n  onUserTurnCompletedDelayMs: number;\n  /**\n   * The time the user stopped speaking.\n   */\n  lastSpeakingTimeMs: number;\n  /**\n   * The ID of the speech handle.\n   */\n  speechId?: string;\n};\n\nexport type RealtimeModelMetricsCachedTokenDetails = {\n  audioTokens: number;\n  textTokens: number;\n  imageTokens: number;\n};\n\nexport type RealtimeModelMetricsInputTokenDetails = {\n  audioTokens: number;\n  textTokens: number;\n  imageTokens: number;\n  cachedTokens: number;\n  cachedTokensDetails?: RealtimeModelMetricsCachedTokenDetails;\n};\n\nexport type RealtimeModelMetricsOutputTokenDetails = {\n  textTokens: number;\n  audioTokens: number;\n  imageTokens: number;\n};\n\nexport type RealtimeModelMetrics = {\n  type: 'realtime_model_metrics';\n  label: string;\n  requestId: string;\n  /**\n   * The timestamp of the response creation.\n   */\n  timestamp: number;\n  /**\n   * The duration of the response from created to done in milliseconds.\n   */\n  durationMs: number;\n  /**\n   * The duration of the session connection in milliseconds (for session-based billing like xAI).\n   */\n  sessionDurationMs?: number;\n  /**\n   * Time to first audio token in milliseconds. -1 if no audio token was sent.\n   */\n  ttftMs: number;\n  /**\n   * Whether the request was cancelled.\n   */\n  cancelled: boolean;\n  /**\n   * The number of input tokens used in the Response, including text and audio tokens.\n   */\n  inputTokens: number;\n  /**\n   * The number of output tokens sent in the Response, including text and audio tokens.\n   */\n  outputTokens: number;\n  /**\n   * The total number of tokens in the Response.\n   */\n  totalTokens: number;\n  /**\n   * The number of tokens per second.\n   */\n  tokensPerSecond: number;\n  /**\n   * Details about the input tokens used in the Response.\n   */\n  inputTokenDetails: RealtimeModelMetricsInputTokenDetails;\n  /**\n   * Details about the output tokens used in the Response.\n   */\n  outputTokenDetails: RealtimeModelMetricsOutputTokenDetails;\n  /** Metadata for model provider and name tracking. */\n  metadata?: MetricsMetadata;\n};\n\nexport type InterruptionMetrics = {\n  type: 'interruption_metrics';\n  timestamp: number;\n  /** Latest RTT time taken to perform inference, in milliseconds. */\n  totalDuration: number;\n  /** Latest time taken by the model side, in milliseconds. */\n  predictionDuration: number;\n  /** Latest total time from onset of speech to final prediction, in milliseconds. */\n  detectionDelay: number;\n  /** Number of interruptions detected (incremental). */\n  numInterruptions: number;\n  /** Number of backchannels detected (incremental). */\n  numBackchannels: number;\n  /** Number of requests sent to the model (incremental). */\n  numRequests: number;\n  metadata?: MetricsMetadata;\n};\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]}

package/dist/metrics/base.d.cts

@@ -1,4 +1,10 @@
-export type AgentMetrics = STTMetrics | LLMMetrics | TTSMetrics | VADMetrics | EOUMetrics | RealtimeModelMetrics;
+export type MetricsMetadata = {
+    /** The provider name (e.g., 'openai', 'anthropic'). */
+    modelProvider?: string;
+    /** The model name (e.g., 'gpt-4o', 'claude-3-5-sonnet'). */
+    modelName?: string;
+};
+export type AgentMetrics = STTMetrics | LLMMetrics | TTSMetrics | VADMetrics | EOUMetrics | RealtimeModelMetrics | InterruptionMetrics;
 export type LLMMetrics = {
     type: 'llm_metrics';
     label: string;
@@ -15,6 +21,8 @@ export type LLMMetrics = {
     totalTokens: number;
     tokensPerSecond: number;
     speechId?: string;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
 };
 export type STTMetrics = {
     type: 'stt_metrics';
@@ -29,10 +37,16 @@ export type STTMetrics = {
      * The duration of the pushed audio in milliseconds.
      */
     audioDurationMs: number;
+    /** Input audio tokens (for token-based billing). */
+    inputTokens?: number;
+    /** Output text tokens (for token-based billing). */
+    outputTokens?: number;
     /**
      * Whether the STT is streaming (e.g using websocket).
      */
     streamed: boolean;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
 };
 export type TTSMetrics = {
     type: 'tts_metrics';
@@ -46,10 +60,17 @@ export type TTSMetrics = {
     /** Generated audio duration in milliseconds. */
     audioDurationMs: number;
     cancelled: boolean;
+    /** Number of characters synthesized (for character-based billing). */
     charactersCount: number;
+    /** Input text tokens (for token-based billing, e.g., OpenAI TTS). */
+    inputTokens?: number;
+    /** Output audio tokens (for token-based billing, e.g., OpenAI TTS). */
+    outputTokens?: number;
     streamed: boolean;
     segmentId?: string;
     speechId?: string;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
 };
 export type VADMetrics = {
     type: 'vad_metrics';
@@ -114,6 +135,10 @@ export type RealtimeModelMetrics = {
      * The duration of the response from created to done in milliseconds.
      */
     durationMs: number;
+    /**
+     * The duration of the session connection in milliseconds (for session-based billing like xAI).
+     */
+    sessionDurationMs?: number;
     /**
      * Time to first audio token in milliseconds. -1 if no audio token was sent.
      */
@@ -146,5 +171,24 @@ export type RealtimeModelMetrics = {
      * Details about the output tokens used in the Response.
      */
     outputTokenDetails: RealtimeModelMetricsOutputTokenDetails;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
+};
+export type InterruptionMetrics = {
+    type: 'interruption_metrics';
+    timestamp: number;
+    /** Latest RTT time taken to perform inference, in milliseconds. */
+    totalDuration: number;
+    /** Latest time taken by the model side, in milliseconds. */
+    predictionDuration: number;
+    /** Latest total time from onset of speech to final prediction, in milliseconds. */
+    detectionDelay: number;
+    /** Number of interruptions detected (incremental). */
+    numInterruptions: number;
+    /** Number of backchannels detected (incremental). */
+    numBackchannels: number;
+    /** Number of requests sent to the model (incremental). */
+    numRequests: number;
+    metadata?: MetricsMetadata;
 };
 //# sourceMappingURL=base.d.ts.map

package/dist/metrics/base.d.ts

@@ -1,4 +1,10 @@
-export type AgentMetrics = STTMetrics | LLMMetrics | TTSMetrics | VADMetrics | EOUMetrics | RealtimeModelMetrics;
+export type MetricsMetadata = {
+    /** The provider name (e.g., 'openai', 'anthropic'). */
+    modelProvider?: string;
+    /** The model name (e.g., 'gpt-4o', 'claude-3-5-sonnet'). */
+    modelName?: string;
+};
+export type AgentMetrics = STTMetrics | LLMMetrics | TTSMetrics | VADMetrics | EOUMetrics | RealtimeModelMetrics | InterruptionMetrics;
 export type LLMMetrics = {
     type: 'llm_metrics';
     label: string;
@@ -15,6 +21,8 @@ export type LLMMetrics = {
     totalTokens: number;
     tokensPerSecond: number;
     speechId?: string;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
 };
 export type STTMetrics = {
     type: 'stt_metrics';
@@ -29,10 +37,16 @@ export type STTMetrics = {
      * The duration of the pushed audio in milliseconds.
      */
     audioDurationMs: number;
+    /** Input audio tokens (for token-based billing). */
+    inputTokens?: number;
+    /** Output text tokens (for token-based billing). */
+    outputTokens?: number;
     /**
      * Whether the STT is streaming (e.g using websocket).
      */
     streamed: boolean;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
 };
 export type TTSMetrics = {
     type: 'tts_metrics';
@@ -46,10 +60,17 @@ export type TTSMetrics = {
     /** Generated audio duration in milliseconds. */
     audioDurationMs: number;
     cancelled: boolean;
+    /** Number of characters synthesized (for character-based billing). */
     charactersCount: number;
+    /** Input text tokens (for token-based billing, e.g., OpenAI TTS). */
+    inputTokens?: number;
+    /** Output audio tokens (for token-based billing, e.g., OpenAI TTS). */
+    outputTokens?: number;
     streamed: boolean;
     segmentId?: string;
     speechId?: string;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
 };
 export type VADMetrics = {
     type: 'vad_metrics';
@@ -114,6 +135,10 @@ export type RealtimeModelMetrics = {
      * The duration of the response from created to done in milliseconds.
      */
     durationMs: number;
+    /**
+     * The duration of the session connection in milliseconds (for session-based billing like xAI).
+     */
+    sessionDurationMs?: number;
     /**
      * Time to first audio token in milliseconds. -1 if no audio token was sent.
      */
@@ -146,5 +171,24 @@ export type RealtimeModelMetrics = {
      * Details about the output tokens used in the Response.
      */
     outputTokenDetails: RealtimeModelMetricsOutputTokenDetails;
+    /** Metadata for model provider and name tracking. */
+    metadata?: MetricsMetadata;
+};
+export type InterruptionMetrics = {
+    type: 'interruption_metrics';
+    timestamp: number;
+    /** Latest RTT time taken to perform inference, in milliseconds. */
+    totalDuration: number;
+    /** Latest time taken by the model side, in milliseconds. */
+    predictionDuration: number;
+    /** Latest total time from onset of speech to final prediction, in milliseconds. */
+    detectionDelay: number;
+    /** Number of interruptions detected (incremental). */
+    numInterruptions: number;
+    /** Number of backchannels detected (incremental). */
+    numBackchannels: number;
+    /** Number of requests sent to the model (incremental). */
+    numRequests: number;
+    metadata?: MetricsMetadata;
 };
 //# sourceMappingURL=base.d.ts.map