@livekit/agents 1.1.0 → 1.2.0

package/src/metrics/model_usage.ts

@@ -0,0 +1,262 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type {
+  AgentMetrics,
+  InterruptionMetrics,
+  LLMMetrics,
+  RealtimeModelMetrics,
+  STTMetrics,
+  TTSMetrics,
+} from './base.js';
+
+export type LLMModelUsage = {
+  type: 'llm_usage';
+  /** The provider name (e.g., 'openai', 'anthropic'). */
+  provider: string;
+  /** The model name (e.g., 'gpt-4o', 'claude-3-5-sonnet'). */
+  model: string;
+  /** Total input tokens. */
+  inputTokens: number;
+  /** Input tokens served from cache. */
+  inputCachedTokens: number;
+  /** Input audio tokens (for multimodal models). */
+  inputAudioTokens: number;
+  /** Cached input audio tokens. */
+  inputCachedAudioTokens: number;
+  /** Input text tokens. */
+  inputTextTokens: number;
+  /** Cached input text tokens. */
+  inputCachedTextTokens: number;
+  /** Input image tokens (for multimodal models). */
+  inputImageTokens: number;
+  /** Cached input image tokens. */
+  inputCachedImageTokens: number;
+  /** Total output tokens. */
+  outputTokens: number;
+  /** Output audio tokens (for multimodal models). */
+  outputAudioTokens: number;
+  /** Output text tokens. */
+  outputTextTokens: number;
+  /** Total session connection duration in milliseconds (for session-based billing like xAI). */
+  sessionDurationMs: number;
+};
+
+export type TTSModelUsage = {
+  type: 'tts_usage';
+  /** The provider name (e.g., 'elevenlabs', 'cartesia'). */
+  provider: string;
+  /** The model name (e.g., 'eleven_turbo_v2', 'sonic'). */
+  model: string;
+  /** Input text tokens (for token-based TTS billing, e.g., OpenAI TTS). */
+  inputTokens: number;
+  /** Output audio tokens (for token-based TTS billing, e.g., OpenAI TTS). */
+  outputTokens: number;
+  /** Number of characters synthesized (for character-based TTS billing). */
+  charactersCount: number;
+  /**
+   * Duration of generated audio in milliseconds.
+   */
+  audioDurationMs: number;
+};
+
+export type STTModelUsage = {
+  type: 'stt_usage';
+  /** The provider name (e.g., 'deepgram', 'assemblyai'). */
+  provider: string;
+  /** The model name (e.g., 'nova-2', 'best'). */
+  model: string;
+  /** Input audio tokens (for token-based STT billing). */
+  inputTokens: number;
+  /** Output text tokens (for token-based STT billing). */
+  outputTokens: number;
+  /** Duration of processed audio in milliseconds. */
+  audioDurationMs: number;
+};
+
+export type InterruptionModelUsage = {
+  type: 'interruption_usage';
+  /** The provider name (e.g., 'livekit'). */
+  provider: string;
+  /** The model name (e.g., 'adaptive interruption'). */
+  model: string;
+  /** Total number of requests sent. */
+  totalRequests: number;
+};
+
+export type ModelUsage = LLMModelUsage | TTSModelUsage | STTModelUsage | InterruptionModelUsage;
+
+export function filterZeroValues<T extends ModelUsage>(usage: T): Partial<T> {
+  const result: Partial<T> = {} as Partial<T>;
+  for (const [key, value] of Object.entries(usage)) {
+    if (value !== 0 && value !== 0.0) {
+      (result as Record<string, unknown>)[key] = value;
+    }
+  }
+  return result;
+}
+
+export class ModelUsageCollector {
+  private llmUsage: Map<string, LLMModelUsage> = new Map();
+  private ttsUsage: Map<string, TTSModelUsage> = new Map();
+  private sttUsage: Map<string, STTModelUsage> = new Map();
+
+  private interruptionUsage: Map<string, InterruptionModelUsage> = new Map();
+
+  /** Extract provider and model from metrics metadata. */
+  private extractProviderModel(
+    metrics: LLMMetrics | STTMetrics | TTSMetrics | RealtimeModelMetrics | InterruptionMetrics,
+  ): [string, string] {
+    let provider = '';
+    let model = '';
+    if (metrics.metadata) {
+      provider = metrics.metadata.modelProvider || '';
+      model = metrics.metadata.modelName || '';
+    }
+    return [provider, model];
+  }
+
+  /** Get or create an LLMModelUsage for the given provider/model combination. */
+  private getLLMUsage(provider: string, model: string): LLMModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.llmUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'llm_usage',
+        provider,
+        model,
+        inputTokens: 0,
+        inputCachedTokens: 0,
+        inputAudioTokens: 0,
+        inputCachedAudioTokens: 0,
+        inputTextTokens: 0,
+        inputCachedTextTokens: 0,
+        inputImageTokens: 0,
+        inputCachedImageTokens: 0,
+        outputTokens: 0,
+        outputAudioTokens: 0,
+        outputTextTokens: 0,
+        sessionDurationMs: 0,
+      };
+      this.llmUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  /** Get or create a TTSModelUsage for the given provider/model combination. */
+  private getTTSUsage(provider: string, model: string): TTSModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.ttsUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'tts_usage',
+        provider,
+        model,
+        inputTokens: 0,
+        outputTokens: 0,
+        charactersCount: 0,
+        audioDurationMs: 0,
+      };
+      this.ttsUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  /** Get or create an STTModelUsage for the given provider/model combination. */
+  private getSTTUsage(provider: string, model: string): STTModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.sttUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'stt_usage',
+        provider,
+        model,
+        inputTokens: 0,
+        outputTokens: 0,
+        audioDurationMs: 0,
+      };
+      this.sttUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  private getInterruptionUsage(provider: string, model: string): InterruptionModelUsage {
+    const key = `${provider}:${model}`;
+    let usage = this.interruptionUsage.get(key);
+    if (!usage) {
+      usage = {
+        type: 'interruption_usage',
+        provider,
+        model,
+        totalRequests: 0,
+      };
+      this.interruptionUsage.set(key, usage);
+    }
+    return usage;
+  }
+
+  /** Collect metrics and aggregate usage by model/provider. */
+  collect(metrics: AgentMetrics): void {
+    if (metrics.type === 'llm_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const usage = this.getLLMUsage(provider, model);
+      usage.inputTokens += metrics.promptTokens;
+      usage.inputCachedTokens += metrics.promptCachedTokens;
+      usage.outputTokens += metrics.completionTokens;
+    } else if (metrics.type === 'realtime_model_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const usage = this.getLLMUsage(provider, model);
+      usage.inputTokens += metrics.inputTokens;
+      usage.inputCachedTokens += metrics.inputTokenDetails.cachedTokens;
+
+      usage.inputTextTokens += metrics.inputTokenDetails.textTokens;
+      usage.inputCachedTextTokens += metrics.inputTokenDetails.cachedTokensDetails?.textTokens ?? 0;
+      usage.inputImageTokens += metrics.inputTokenDetails.imageTokens;
+      usage.inputCachedImageTokens +=
+        metrics.inputTokenDetails.cachedTokensDetails?.imageTokens ?? 0;
+      usage.inputAudioTokens += metrics.inputTokenDetails.audioTokens;
+      usage.inputCachedAudioTokens +=
+        metrics.inputTokenDetails.cachedTokensDetails?.audioTokens ?? 0;
+
+      usage.outputTextTokens += metrics.outputTokenDetails.textTokens;
+      usage.outputAudioTokens += metrics.outputTokenDetails.audioTokens;
+      usage.outputTokens += metrics.outputTokens;
+      usage.sessionDurationMs += metrics.sessionDurationMs ?? 0;
+    } else if (metrics.type === 'tts_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const ttsUsage = this.getTTSUsage(provider, model);
+      ttsUsage.inputTokens += metrics.inputTokens ?? 0;
+      ttsUsage.outputTokens += metrics.outputTokens ?? 0;
+      ttsUsage.charactersCount += metrics.charactersCount;
+      ttsUsage.audioDurationMs += metrics.audioDurationMs;
+    } else if (metrics.type === 'stt_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const sttUsage = this.getSTTUsage(provider, model);
+      sttUsage.inputTokens += metrics.inputTokens ?? 0;
+      sttUsage.outputTokens += metrics.outputTokens ?? 0;
+      sttUsage.audioDurationMs += metrics.audioDurationMs;
+    } else if (metrics.type === 'interruption_metrics') {
+      const [provider, model] = this.extractProviderModel(metrics);
+      const usage = this.getInterruptionUsage(provider, model);
+      usage.totalRequests += metrics.numRequests;
+    }
+    // VAD and EOU metrics are not aggregated for usage tracking.
+  }
+
+  flatten(): ModelUsage[] {
+    const result: ModelUsage[] = [];
+    for (const u of this.llmUsage.values()) {
+      result.push({ ...u });
+    }
+    for (const u of this.ttsUsage.values()) {
+      result.push({ ...u });
+    }
+    for (const u of this.sttUsage.values()) {
+      result.push({ ...u });
+    }
+    for (const u of this.interruptionUsage.values()) {
+      result.push({ ...u });
+    }
+    return result;
+  }
+}

package/src/metrics/usage_collector.ts

@@ -1,26 +1,37 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import { log } from '../log.js';
 import type { AgentMetrics } from './base.js';
 
+/**
+ * @deprecated Use LLMModelUsage, TTSModelUsage, or STTModelUsage instead.
+ * These new types provide per-model/provider usage aggregation for more detailed tracking.
+ */
 export interface UsageSummary {
   llmPromptTokens: number;
   llmPromptCachedTokens: number;
   llmCompletionTokens: number;
   ttsCharactersCount: number;
-  sttAudioDuration: number;
+  sttAudioDurationMs: number;
 }
 
+/**
+ * @deprecated Use ModelUsageCollector instead.
+ * ModelUsageCollector provides per-model/provider usage aggregation for more detailed tracking.
+ */
 export class UsageCollector {
   private summary: UsageSummary;
+  private logger = log();
 
   constructor() {
+    this.logger.warn('UsageCollector is deprecated. Use ModelUsageCollector instead.');
     this.summary = {
       llmPromptTokens: 0,
       llmPromptCachedTokens: 0,
       llmCompletionTokens: 0,
       ttsCharactersCount: 0,
-      sttAudioDuration: 0,
+      sttAudioDurationMs: 0,
     };
   }
 
@@ -36,7 +47,7 @@ export class UsageCollector {
     } else if (metrics.type === 'tts_metrics') {
       this.summary.ttsCharactersCount += metrics.charactersCount;
     } else if (metrics.type === 'stt_metrics') {
-      this.summary.sttAudioDuration += metrics.audioDuration;
+      this.summary.sttAudioDurationMs += metrics.audioDurationMs;
     }
   }
 

package/src/metrics/utils.ts

@@ -13,7 +13,7 @@ export const logMetrics = (metrics: AgentMetrics) => {
   if (metrics.type === 'llm_metrics') {
     logger
       .child({
-        ttft: roundTwoDecimals(metrics.ttft),
+        ttftMs: roundTwoDecimals(metrics.ttftMs),
         inputTokens: metrics.promptTokens,
         promptCachedTokens: metrics.promptCachedTokens,
         outputTokens: metrics.completionTokens,
@@ -23,7 +23,7 @@ export const logMetrics = (metrics: AgentMetrics) => {
   } else if (metrics.type === 'realtime_model_metrics') {
     logger
       .child({
-        ttft: roundTwoDecimals(metrics.ttft),
+        ttftMs: roundTwoDecimals(metrics.ttftMs),
         input_tokens: metrics.inputTokens,
         cached_input_tokens: metrics.inputTokenDetails.cachedTokens,
         output_tokens: metrics.outputTokens,
@@ -34,22 +34,42 @@ export const logMetrics = (metrics: AgentMetrics) => {
   } else if (metrics.type === 'tts_metrics') {
     logger
       .child({
-        ttfb: roundTwoDecimals(metrics.ttfb),
-        audioDuration: metrics.audioDuration,
+        ttfbMs: roundTwoDecimals(metrics.ttfbMs),
+        audioDurationMs: Math.round(metrics.audioDurationMs),
       })
       .info('TTS metrics');
   } else if (metrics.type === 'eou_metrics') {
     logger
       .child({
-        end_of_utterance_delay: roundTwoDecimals(metrics.endOfUtteranceDelay),
-        transcription_delay: roundTwoDecimals(metrics.transcriptionDelay),
+        endOfUtteranceDelayMs: roundTwoDecimals(metrics.endOfUtteranceDelayMs),
+        transcriptionDelayMs: roundTwoDecimals(metrics.transcriptionDelayMs),
+        onUserTurnCompletedDelayMs: roundTwoDecimals(metrics.onUserTurnCompletedDelayMs),
       })
       .info('EOU metrics');
+  } else if (metrics.type === 'vad_metrics') {
+    logger
+      .child({
+        idleTimeMs: Math.round(metrics.idleTimeMs),
+        inferenceDurationTotalMs: Math.round(metrics.inferenceDurationTotalMs),
+        inferenceCount: metrics.inferenceCount,
+      })
+      .info('VAD metrics');
   } else if (metrics.type === 'stt_metrics') {
     logger
       .child({
-        audioDuration: metrics.audioDuration,
+        audioDurationMs: Math.round(metrics.audioDurationMs),
       })
       .info('STT metrics');
+  } else if (metrics.type === 'interruption_metrics') {
+    logger
+      .child({
+        totalDurationMs: roundTwoDecimals(metrics.totalDuration),
+        predictionDurationMs: roundTwoDecimals(metrics.predictionDuration),
+        detectionDelayMs: roundTwoDecimals(metrics.detectionDelay),
+        numInterruptions: metrics.numInterruptions,
+        numBackchannels: metrics.numBackchannels,
+        numRequests: metrics.numRequests,
+      })
+      .info('Interruption metrics');
   }
 };

package/src/stream/deferred_stream.test.ts

@@ -522,11 +522,11 @@ describe('DeferredReadableStream', () => {
     expect(() => deferred.setSource(source2)).toThrow('Stream source already set');
   });
 
-  it('should throw error when trying to detach source before setting it', async () => {
+  it('should safely detach source before setting it (no-op)', async () => {
     const deferred = new DeferredReadableStream<string>();
 
-    // Attempting to detach source before setting it should throw
-    await expect(deferred.detachSource()).rejects.toThrow('Source not set');
+    // Attempting to detach source before setting it should be a no-op, not throw
+    await expect(deferred.detachSource()).resolves.toBeUndefined();
   });
 
   it('read returns undefined as soon as reader is cancelled', async () => {

package/src/stream/deferred_stream.ts

@@ -9,18 +9,29 @@ import type {
 import { IdentityTransform } from './identity_transform.js';
 
 /**
- * Check if error is related to reader.read after release lock
+ * Check if error is related to stream cleanup operations.
+ *
+ * These errors are expected when calling reader.read() after releaseLock()
+ * or when writing to already closed streams during cleanup:
  *
  * Invalid state: Releasing reader
  * Invalid state: The reader is not attached to a stream
+ * Invalid state: Controller is already closed
+ * Invalid state: WritableStream is closed
  */
 export function isStreamReaderReleaseError(e: unknown) {
   const allowedMessages = [
     'Invalid state: Releasing reader',
     'Invalid state: The reader is not attached to a stream',
+    'Controller is already closed',
+    'WritableStream is closed',
+    // bun
+    'Stream reader cancelled via releaseLock()',
+    // deno
+    'The reader was released.',
   ];
 
-  if (e instanceof TypeError) {
+  if (e instanceof TypeError || (e instanceof Error && e.name === 'AbortError')) {
     return allowedMessages.some((message) => e.message.includes(message));
   }
 
@@ -52,32 +63,42 @@ export class DeferredReadableStream<T> {
       throw new Error('Stream source already set');
     }
 
-    this.sourceReader = source.getReader();
-    this.pump();
+    const sourceReader = source.getReader();
+    this.sourceReader = sourceReader;
+    void this.pump(sourceReader);
   }
 
-  private async pump() {
+  private async pump(sourceReader: ReadableStreamDefaultReader<T>) {
     let sourceError: unknown;
 
     try {
       while (true) {
-        const { done, value } = await this.sourceReader!.read();
+        const { done, value } = await sourceReader.read();
         if (done) break;
         await this.writer.write(value);
       }
     } catch (e) {
-      // skip source detach related errors
+      // skip stream cleanup related errors
       if (isStreamReaderReleaseError(e)) return;
+
       sourceError = e;
     } finally {
       // any other error from source will be propagated to the consumer
       if (sourceError) {
-        this.writer.abort(sourceError);
+        try {
+          await this.writer.abort(sourceError);
+        } catch (e) {
+          // ignore if writer is already closed
+        }
         return;
       }
 
       // release lock so this.stream.getReader().read() will terminate with done: true
-      this.writer.releaseLock();
+      try {
+        this.writer.releaseLock();
+      } catch (e) {
+        // ignore if writer lock is already released
+      }
 
       // we only close the writable stream after done
       try {
@@ -98,13 +119,24 @@ export class DeferredReadableStream<T> {
    */
   async detachSource() {
     if (!this.isSourceSet) {
-      throw new Error('Source not set');
+      // No-op if source was never set - this is a common case during cleanup
+      return;
     }
 
+    const sourceReader = this.sourceReader!;
+    // Clear source first so future setSource() calls can reattach cleanly.
+    this.sourceReader = undefined;
+
     // release lock will make any pending read() throw TypeError
     // which are expected, and we intentionally catch those error
     // using isStreamReaderReleaseError
     // this will unblock any pending read() inside the async for loop
-    this.sourceReader!.releaseLock();
+    try {
+      sourceReader.releaseLock();
+    } catch (e) {
+      if (!isStreamReaderReleaseError(e)) {
+        throw e;
+      }
+    }
   }
 }

package/src/stream/index.ts

@@ -4,4 +4,5 @@
 export { DeferredReadableStream } from './deferred_stream.js';
 export { IdentityTransform } from './identity_transform.js';
 export { mergeReadableStreams } from './merge_readable_streams.js';
+export { MultiInputStream } from './multi_input_stream.js';
 export { createStreamChannel, type StreamChannel } from './stream_channel.js';