@livekit/agents 1.0.48 → 1.1.0-dev.0

package/src/metrics/model_usage.ts

@@ -0,0 +1,262 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type {
+  AgentMetrics,
+  InterruptionMetrics,
+  LLMMetrics,
+  RealtimeModelMetrics,
+  STTMetrics,
+  TTSMetrics,
+} from './base.js';
+
+export type LLMModelUsage = {
+  type: 'llm_usage';
+  /** The provider name (e.g., 'openai', 'anthropic'). */
+  provider: string;
+  /** The model name (e.g., 'gpt-4o', 'claude-3-5-sonnet'). */
+  model: string;
+  /** Total input tokens. */
+  inputTokens: number;
+  /** Input tokens served from cache. */
+  inputCachedTokens: number;
+  /** Input audio tokens (for multimodal models). */
+  inputAudioTokens: number;
+  /** Cached input audio tokens. */
+  inputCachedAudioTokens: number;
+  /** Input text tokens. */
+  inputTextTokens: number;
+  /** Cached input text tokens. */
+  inputCachedTextTokens: number;
+  /** Input image tokens (for multimodal models). */
+  inputImageTokens: number;
+  /** Cached input image tokens. */
+  inputCachedImageTokens: number;
+  /** Total output tokens. */
+  outputTokens: number;
+  /** Output audio tokens (for multimodal models). */
+  outputAudioTokens: number;
+  /** Output text tokens. */
+  outputTextTokens: number;
+  /** Total session connection duration in milliseconds (for session-based billing like xAI). */
+  sessionDurationMs: number;
+};
+
+export type TTSModelUsage = {
+  type: 'tts_usage';
+  /** The provider name (e.g., 'elevenlabs', 'cartesia'). */
+  provider: string;
+  /** The model name (e.g., 'eleven_turbo_v2', 'sonic'). */
+  model: string;
+  /** Input text tokens (for token-based TTS billing, e.g., OpenAI TTS). */
+  inputTokens: number;
+  /** Output audio tokens (for token-based TTS billing, e.g., OpenAI TTS). */
+  outputTokens: number;
+  /** Number of characters synthesized (for character-based TTS billing). */
+  charactersCount: number;
+  /**
+   * Duration of generated audio in milliseconds.
+   */
+  audioDurationMs: number;
+};
+
+export type STTModelUsage = {
+  type: 'stt_usage';
+  /** The provider name (e.g., 'deepgram', 'assemblyai'). */
+  provider: string;
+  /** The model name (e.g., 'nova-2', 'best'). */
+  model: string;
+  /** Input audio tokens (for token-based STT billing). */
+  inputTokens: number;
+  /** Output text tokens (for token-based STT billing). */
+  outputTokens: number;
+  /** Duration of processed audio in milliseconds. */
+  audioDurationMs: number;
+};
+
+export type InterruptionModelUsage = {
+  type: 'interruption_usage';
+  /** The provider name (e.g., 'livekit'). */
+  provider: string;
+  /** The model name (e.g., 'adaptive interruption'). */
+  model: string;
+  /** Total number of requests sent. */
+  totalRequests: number;
+};
+
+export type ModelUsage = LLMModelUsage | TTSModelUsage | STTModelUsage | InterruptionModelUsage;
+
+export function filterZeroValues<T extends ModelUsage>(usage: T): Partial<T> {
+  const result: Partial<T> = {} as Partial<T>;
+  for (const [key, value] of Object.entries(usage)) {
+    if (value !== 0 && value !== 0.0) {
+      (result as Record<string, unknown>)[key] = value;
+    }
+  }
+  return result;
+}
+
+export class ModelUsageCollector {
+  private llmUsage: Map<string, LLMModelUsage> = new Map();
+  private ttsUsage: Map<string, TTSModelUsage> = new Map();
+  private sttUsage: Map<string, STTModelUsage> = new Map();
+
+  private interruptionUsage: Map<string, InterruptionModelUsage> = new Map();
+
+  /** Extract provider and model from metrics metadata. */
+  private extractProviderModel(
+    metrics: LLMMetrics | STTMetrics | TTSMetrics | RealtimeModelMetrics | InterruptionMetrics,
+  ): [string, string] {
+    let provider = '';
+    let model = '';
+    if (metrics.metadata) {
+      provider = metrics.metadata.modelProvider || '';
+      model = metrics.metadata.modelName || '';
+    }
+    return [provider, model];
+  }
+
+  /** Get or create an LLMModelUsage for the given provider/model combination. */
+  private getLLMUsage(provider: string, model: string): LLMModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.llmUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'llm_usage',
+        provider,
+        model,
+        inputTokens: 0,
+        inputCachedTokens: 0,
+        inputAudioTokens: 0,
+        inputCachedAudioTokens: 0,
+        inputTextTokens: 0,
+        inputCachedTextTokens: 0,
+        inputImageTokens: 0,
+        inputCachedImageTokens: 0,
+        outputTokens: 0,
+        outputAudioTokens: 0,
+        outputTextTokens: 0,
+        sessionDurationMs: 0,
+      };
+      this.llmUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  /** Get or create a TTSModelUsage for the given provider/model combination. */
+  private getTTSUsage(provider: string, model: string): TTSModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.ttsUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'tts_usage',
+        provider,
+        model,
+        inputTokens: 0,
+        outputTokens: 0,
+        charactersCount: 0,
+        audioDurationMs: 0,
+      };
+      this.ttsUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  /** Get or create an STTModelUsage for the given provider/model combination. */
+  private getSTTUsage(provider: string, model: string): STTModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.sttUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'stt_usage',
+        provider,
+        model,
+        inputTokens: 0,
+        outputTokens: 0,
+        audioDurationMs: 0,
+      };
+      this.sttUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  private getInterruptionUsage(provider: string, model: string): InterruptionModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.interruptionUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'interruption_usage',
+        provider,
+        model,
+        totalRequests: 0,
+      };
+      this.interruptionUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  /** Collect metrics and aggregate usage by model/provider. */
+  collect(metrics: AgentMetrics): void {
+    if (metrics.type === 'llm_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const usage = this.getLLMUsage(provider, model);
+      usage.inputTokens += metrics.promptTokens;
+      usage.inputCachedTokens += metrics.promptCachedTokens;
+      usage.outputTokens += metrics.completionTokens;
+    } else if (metrics.type === 'realtime_model_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const usage = this.getLLMUsage(provider, model);
+      usage.inputTokens += metrics.inputTokens;
+      usage.inputCachedTokens += metrics.inputTokenDetails.cachedTokens;
+
+      usage.inputTextTokens += metrics.inputTokenDetails.textTokens;
+      usage.inputCachedTextTokens += metrics.inputTokenDetails.cachedTokensDetails?.textTokens ?? 0;
+      usage.inputImageTokens += metrics.inputTokenDetails.imageTokens;
+      usage.inputCachedImageTokens +=
+        metrics.inputTokenDetails.cachedTokensDetails?.imageTokens ?? 0;
+      usage.inputAudioTokens += metrics.inputTokenDetails.audioTokens;
+      usage.inputCachedAudioTokens +=
+        metrics.inputTokenDetails.cachedTokensDetails?.audioTokens ?? 0;
+
+      usage.outputTextTokens += metrics.outputTokenDetails.textTokens;
+      usage.outputAudioTokens += metrics.outputTokenDetails.audioTokens;
+      usage.outputTokens += metrics.outputTokens;
+      usage.sessionDurationMs += metrics.sessionDurationMs ?? 0;
+    } else if (metrics.type === 'tts_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const ttsUsage = this.getTTSUsage(provider, model);
+      ttsUsage.inputTokens += metrics.inputTokens ?? 0;
+      ttsUsage.outputTokens += metrics.outputTokens ?? 0;
+      ttsUsage.charactersCount += metrics.charactersCount;
+      ttsUsage.audioDurationMs += metrics.audioDurationMs;
+    } else if (metrics.type === 'stt_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const sttUsage = this.getSTTUsage(provider, model);
+      sttUsage.inputTokens += metrics.inputTokens ?? 0;
+      sttUsage.outputTokens += metrics.outputTokens ?? 0;
+      sttUsage.audioDurationMs += metrics.audioDurationMs;
+    } else if (metrics.type === 'interruption_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const usage = this.getInterruptionUsage(provider, model);
+      usage.totalRequests += metrics.numRequests;
+    }
+    // VAD and EOU metrics are not aggregated for usage tracking.
+  }
+
+  flatten(): ModelUsage[] {
+    const result: ModelUsage[] = [];
+    for (const u of this.llmUsage.values()) {
+      result.push({ ...u });
+    }
+    for (const u of this.ttsUsage.values()) {
+      result.push({ ...u });
+    }
+    for (const u of this.sttUsage.values()) {
+      result.push({ ...u });
+    }
+    for (const u of this.interruptionUsage.values()) {
+      result.push({ ...u });
+    }
+    return result;
+  }
+}

package/src/metrics/usage_collector.ts

@@ -1,8 +1,13 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import { log } from '../log.js';
 import type { AgentMetrics } from './base.js';
 
+/**
+ * @deprecated Use LLMModelUsage, TTSModelUsage, or STTModelUsage instead.
+ * These new types provide per-model/provider usage aggregation for more detailed tracking.
+ */
 export interface UsageSummary {
   llmPromptTokens: number;
   llmPromptCachedTokens: number;
@@ -11,10 +16,16 @@ export interface UsageSummary {
   sttAudioDurationMs: number;
 }
 
+/**
+ * @deprecated Use ModelUsageCollector instead.
+ * ModelUsageCollector provides per-model/provider usage aggregation for more detailed tracking.
+ */
 export class UsageCollector {
   private summary: UsageSummary;
+  private logger = log();
 
   constructor() {
+    this.logger.warn('UsageCollector is deprecated. Use ModelUsageCollector instead.');
     this.summary = {
       llmPromptTokens: 0,
       llmPromptCachedTokens: 0,

package/src/metrics/utils.ts

@@ -60,5 +60,16 @@ export const logMetrics = (metrics: AgentMetrics) => {
         audioDurationMs: Math.round(metrics.audioDurationMs),
       })
       .info('STT metrics');
+  } else if (metrics.type === 'interruption_metrics') {
+    logger
+      .child({
+        totalDurationMs: roundTwoDecimals(metrics.totalDuration),
+        predictionDurationMs: roundTwoDecimals(metrics.predictionDuration),
+        detectionDelayMs: roundTwoDecimals(metrics.detectionDelay),
+        numInterruptions: metrics.numInterruptions,
+        numBackchannels: metrics.numBackchannels,
+        numRequests: metrics.numRequests,
+      })
+      .info('Interruption metrics');
   }
 };

package/src/stream/multi_input_stream.test.ts

@@ -2,7 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { ReadableStream } from 'node:stream/web';
-import { describe, expect, it } from 'vitest';
+import { beforeAll, describe, expect, it } from 'vitest';
+import { initializeLogger } from '../log.js';
 import { delay } from '../utils.js';
 import { MultiInputStream } from './multi_input_stream.js';
 
@@ -16,6 +17,10 @@ function streamFrom<T>(values: T[]): ReadableStream<T> {
 }
 
 describe('MultiInputStream', () => {
+  beforeAll(() => {
+    initializeLogger({ pretty: false });
+  });
+
   // ---------------------------------------------------------------------------
   // Basic functionality
   // ---------------------------------------------------------------------------

package/src/stream/stream_channel.ts

@@ -4,14 +4,16 @@
 import type { ReadableStream } from 'node:stream/web';
 import { IdentityTransform } from './identity_transform.js';
 
-export interface StreamChannel<T> {
+export interface StreamChannel<T, E extends Error = Error> {
   write(chunk: T): Promise<void>;
   close(): Promise<void>;
   stream(): ReadableStream<T>;
+  abort(error: E): Promise<void>;
   readonly closed: boolean;
+  addStreamInput(stream: ReadableStream<T>): void;
 }
 
-export function createStreamChannel<T>(): StreamChannel<T> {
+export function createStreamChannel<T, E extends Error = Error>(): StreamChannel<T, E> {
   const transform = new IdentityTransform<T>();
   const writer = transform.writable.getWriter();
   let isClosed = false;
@@ -19,6 +21,36 @@ export function createStreamChannel<T>(): StreamChannel<T> {
   return {
     write: (chunk: T) => writer.write(chunk),
     stream: () => transform.readable,
+    abort: async (error: E) => {
+      if (isClosed) return;
+      isClosed = true;
+      try {
+        await writer.abort(error);
+      } catch (e) {
+        if (e instanceof Error && e.name === 'TypeError') return;
+        throw e;
+      }
+    },
+    addStreamInput: (newInputStream) => {
+      if (isClosed) return;
+      const reader = newInputStream.getReader();
+      (async () => {
+        try {
+          while (!isClosed) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            await writer.write(value);
+          }
+        } catch (err) {
+          if (!isClosed) {
+            isClosed = true;
+            await writer.abort(err as E);
+          }
+        } finally {
+          reader.releaseLock();
+        }
+      })().catch(() => {});
+    },
     close: async () => {
       try {
         const result = await writer.close();

package/src/stt/stt.ts

@@ -66,6 +66,10 @@ export interface SpeechData {
 export interface RecognitionUsage {
   /** Duration of the audio that was recognized in seconds. */
   audioDuration: number;
+  /** Input audio tokens (for token-based STT billing). */
+  inputTokens?: number;
+  /** Output text tokens (for token-based STT billing). */
+  outputTokens?: number;
 }
 
 /** SpeechEvent is a packet of speech-to-text data. */
@@ -128,6 +132,30 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCal
     return this.#capabilities;
   }
 
+  /**
+   * Get the model name/identifier for this STT instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model(): string {
+    return 'unknown';
+  }
+
+  /**
+   * Get the provider name for this STT instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider(): string {
+    return 'unknown';
+  }
+
   /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
   async recognize(frame: AudioBuffer, abortSignal?: AbortSignal): Promise<SpeechEvent> {
     const startTime = process.hrtime.bigint();
@@ -141,6 +169,10 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCal
       label: this.label,
       audioDurationMs: Math.round(calculateAudioDurationSeconds(frame) * 1000),
       streamed: false,
+      metadata: {
+        modelProvider: this.provider,
+        modelName: this.model,
+      },
     });
     return event;
   }
@@ -302,7 +334,13 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
         durationMs: 0,
         label: this.#stt.label,
         audioDurationMs: Math.round(event.recognitionUsage!.audioDuration * 1000),
+        inputTokens: event.recognitionUsage!.inputTokens ?? 0,
+        outputTokens: event.recognitionUsage!.outputTokens ?? 0,
         streamed: true,
+        metadata: {
+          modelProvider: this.#stt.provider,
+          modelName: this.#stt.model,
+        },
       };
       this.#stt.emit('metrics_collected', metrics);
     }

package/src/telemetry/otel_http_exporter.ts

@@ -58,6 +58,16 @@ export class SimpleOTLPHttpLogExporter {
   private readonly config: SimpleOTLPHttpLogExporterConfig;
   private jwt: string | null = null;
 
+  private static readonly FORCE_DOUBLE_KEYS = new Set([
+    'transcriptConfidence',
+    'transcriptionDelay',
+    'endOfTurnDelay',
+    'onUserTurnCompletedDelay',
+    'llmNodeTtft',
+    'ttsNodeTtfb',
+    'e2eLatency',
+  ]);
+
   constructor(config: SimpleOTLPHttpLogExporterConfig) {
     this.config = config;
   }
@@ -72,6 +82,7 @@ export class SimpleOTLPHttpLogExporter {
 
     const endpoint = `https://${this.config.cloudHostname}/observability/logs/otlp/v0`;
     const payload = this.buildPayload(records);
+    const payloadJson = JSON.stringify(payload);
 
     const response = await fetch(endpoint, {
       method: 'POST',
@@ -79,7 +90,7 @@ export class SimpleOTLPHttpLogExporter {
         Authorization: `Bearer ${this.jwt}`,
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify(payload),
+      body: payloadJson,
     });
 
     if (!response.ok) {
@@ -160,11 +171,11 @@ export class SimpleOTLPHttpLogExporter {
   ): Array<{ key: string; value: unknown }> {
     return Object.entries(attrs).map(([key, value]) => ({
       key,
-      value: this.convertValue(value),
+      value: this.convertValue(value, key),
     }));
   }
 
-  private convertValue(value: unknown): unknown {
+  private convertValue(value: unknown, path: string = ''): unknown {
     if (value === null || value === undefined) {
       return { stringValue: '' };
     }
@@ -172,20 +183,32 @@ export class SimpleOTLPHttpLogExporter {
       return { stringValue: value };
     }
     if (typeof value === 'number') {
+      const leafKey =
+        path
+          .split('.')
+          .pop()
+          ?.replace(/\[\d+\]$/, '') ?? path;
+      if (SimpleOTLPHttpLogExporter.FORCE_DOUBLE_KEYS.has(leafKey)) {
+        return { doubleValue: value };
+      }
       return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
     }
     if (typeof value === 'boolean') {
       return { boolValue: value };
     }
     if (Array.isArray(value)) {
-      return { arrayValue: { values: value.map((v) => this.convertValue(v)) } };
+      return {
+        arrayValue: {
+          values: value.map((v, i) => this.convertValue(v, `${path}[${i}]`)),
+        },
+      };
     }
     if (typeof value === 'object') {
       return {
         kvlistValue: {
           values: Object.entries(value as Record<string, unknown>).map(([k, v]) => ({
             key: k,
-            value: this.convertValue(v),
+            value: this.convertValue(v, path ? `${path}.${k}` : k),
           })),
         },
       };

package/src/telemetry/trace_types.ts

@@ -33,6 +33,7 @@ export const ATTR_PROVIDER_TOOLS = 'lk.provider_tools';
 export const ATTR_TOOL_SETS = 'lk.tool_sets';
 export const ATTR_RESPONSE_TEXT = 'lk.response.text';
 export const ATTR_RESPONSE_FUNCTION_CALLS = 'lk.response.function_calls';
+/** Time to first token in seconds. */
 export const ATTR_RESPONSE_TTFT = 'lk.response.ttft';
 
 // function tool
@@ -46,6 +47,7 @@ export const ATTR_FUNCTION_TOOL_OUTPUT = 'lk.function_tool.output';
 export const ATTR_TTS_INPUT_TEXT = 'lk.input_text';
 export const ATTR_TTS_STREAMING = 'lk.tts.streaming';
 export const ATTR_TTS_LABEL = 'lk.tts.label';
+/** Time to first byte in seconds. */
 export const ATTR_RESPONSE_TTFB = 'lk.response.ttfb';
 
 // eou detection
@@ -58,18 +60,26 @@ export const ATTR_TRANSCRIPT_CONFIDENCE = 'lk.transcript_confidence';
 export const ATTR_TRANSCRIPTION_DELAY = 'lk.transcription_delay';
 export const ATTR_END_OF_TURN_DELAY = 'lk.end_of_turn_delay';
 
+// Adaptive Interruption attributes
+export const ATTR_IS_INTERRUPTION = 'lk.is_interruption';
+export const ATTR_INTERRUPTION_PROBABILITY = 'lk.interruption.probability';
+export const ATTR_INTERRUPTION_TOTAL_DURATION = 'lk.interruption.total_duration';
+export const ATTR_INTERRUPTION_PREDICTION_DURATION = 'lk.interruption.prediction_duration';
+export const ATTR_INTERRUPTION_DETECTION_DELAY = 'lk.interruption.detection_delay';
+
 // metrics
 export const ATTR_LLM_METRICS = 'lk.llm_metrics';
 export const ATTR_TTS_METRICS = 'lk.tts_metrics';
 export const ATTR_REALTIME_MODEL_METRICS = 'lk.realtime_model_metrics';
 
-// latency span attributes
+/** End-to-end latency in seconds. */
 export const ATTR_E2E_LATENCY = 'lk.e2e_latency';
 
 // OpenTelemetry GenAI attributes
 // OpenTelemetry specification: https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/
 export const ATTR_GEN_AI_OPERATION_NAME = 'gen_ai.operation.name';
 export const ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model';
+/** The provider name (e.g., 'openai', 'anthropic'). */
 export const ATTR_GEN_AI_PROVIDER_NAME = 'gen_ai.provider.name';
 export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens';
 export const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens';
@@ -97,10 +107,3 @@ export const ATTR_EXCEPTION_MESSAGE = 'exception.message';
 
 // Platform-specific attributes
 export const ATTR_LANGFUSE_COMPLETION_START_TIME = 'langfuse.observation.completion_start_time';
-
-// Adaptive Interruption attributes
-export const ATTR_IS_INTERRUPTION = 'lk.is_interruption';
-export const ATTR_INTERRUPTION_PROBABILITY = 'lk.interruption.probability';
-export const ATTR_INTERRUPTION_TOTAL_DURATION = 'lk.interruption.total_duration';
-export const ATTR_INTERRUPTION_PREDICTION_DURATION = 'lk.interruption.prediction_duration';
-export const ATTR_INTERRUPTION_DETECTION_DELAY = 'lk.interruption.detection_delay';