@livekit/agents 0.5.2 → 0.6.1

package/src/index.ts

@@ -11,6 +11,7 @@
  */
 import * as cli from './cli.js';
 import * as llm from './llm/index.js';
+import * as metrics from './metrics/index.js';
 import * as multimodal from './multimodal/index.js';
 import * as pipeline from './pipeline/index.js';
 import * as stt from './stt/index.js';
@@ -28,4 +29,4 @@ export * from './generator.js';
 export * from './audio.js';
 export * from './transcription.js';
 
-export { cli, stt, tts, llm, pipeline, multimodal, tokenize };
+export { cli, stt, tts, llm, pipeline, multimodal, tokenize, metrics };

package/src/job.ts

@@ -190,15 +190,15 @@ export class JobContext {
   /** @internal */
   onParticipantConnected(p: RemoteParticipant) {
     for (const callback of this.#participantEntrypoints) {
-      if (this.#participantTasks[p.identity]?.callback == callback) {
+      if (this.#participantTasks[p.identity!]?.callback == callback) {
         this.#logger.warn(
           'a participant has joined before a prior prticipant task matching the same identity has finished:',
           p.identity,
         );
       }
       const result = callback(this, p);
-      result.finally(() => delete this.#participantTasks[p.identity]);
-      this.#participantTasks[p.identity] = { callback, result };
+      result.finally(() => delete this.#participantTasks[p.identity!]);
+      this.#participantTasks[p.identity!] = { callback, result };
     }
   }
 

package/src/llm/index.ts

@@ -25,6 +25,8 @@ export {
   type CompletionUsage,
   type Choice,
   type ChatChunk,
+  type LLMCallbacks,
+  LLMEvent,
   LLM,
   LLMStream,
 } from './llm.js';

package/src/llm/llm.ts

@@ -1,6 +1,9 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import { EventEmitter } from 'node:events';
+import type { LLMMetrics } from '../metrics/base.js';
 import { AsyncIterableQueue } from '../utils.js';
 import type { ChatContext, ChatRole } from './chat_context.js';
 import type { FunctionCallInfo, FunctionContext } from './function_context.js';
@@ -28,7 +31,15 @@ export interface ChatChunk {
   usage?: CompletionUsage;
 }
 
-export abstract class LLM {
+export enum LLMEvent {
+  METRICS_COLLECTED,
+}
+
+export type LLMCallbacks = {
+  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+};
+
+export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {
   /**
    * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
    */
@@ -48,16 +59,56 @@ export abstract class LLM {
 }
 
 export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+  protected output = new AsyncIterableQueue<ChatChunk>();
   protected queue = new AsyncIterableQueue<ChatChunk>();
   protected closed = false;
   protected _functionCalls: FunctionCallInfo[] = [];
+  abstract label: string;
 
+  #llm: LLM;
   #chatCtx: ChatContext;
   #fncCtx?: FunctionContext;
 
-  constructor(chatCtx: ChatContext, fncCtx?: FunctionContext) {
+  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {
+    this.#llm = llm;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.monitorMetrics();
+  }
+
+  protected async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    let ttft: bigint | undefined;
+    let requestId = '';
+    let usage: CompletionUsage | undefined;
+
+    for await (const ev of this.queue) {
+      this.output.put(ev);
+      requestId = ev.requestId;
+      if (!ttft) {
+        ttft = process.hrtime.bigint() - startTime;
+      }
+      if (ev.usage) {
+        usage = ev.usage;
+      }
+    }
+    this.output.close();
+
+    const duration = process.hrtime.bigint() - startTime;
+    const metrics: LLMMetrics = {
+      timestamp: Date.now(),
+      requestId,
+      ttft: Math.trunc(Number(ttft! / BigInt(1000000))),
+      duration: Math.trunc(Number(duration / BigInt(1000000))),
+      cancelled: false, // XXX(nbsp)
+      label: this.label,
+      completionTokens: usage?.completionTokens || 0,
+      promptTokens: usage?.promptTokens || 0,
+      totalTokens: usage?.totalTokens || 0,
+      tokensPerSecond:
+        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),
+    };
+    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);
   }
 
   /** List of called functions from this stream. */
@@ -88,10 +139,11 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   }
 
   next(): Promise<IteratorResult<ChatChunk>> {
-    return this.queue.next();
+    return this.output.next();
   }
 
   close() {
+    this.output.close();
     this.queue.close();
     this.closed = true;
   }

package/src/metrics/base.ts

@@ -0,0 +1,127 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export interface LLMMetrics {
+  requestId: string;
+  timestamp: number;
+  ttft: number;
+  duration: number;
+  label: string;
+  cancelled: boolean;
+  completionTokens: number;
+  promptTokens: number;
+  totalTokens: number;
+  tokensPerSecond: number;
+  error?: Error;
+}
+
+export interface STTMetrics {
+  requestId: string;
+  timestamp: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface TTSMetrics {
+  requestId: string;
+  timestamp: number;
+  ttfb: number;
+  duration: number;
+  label: string;
+  audioDuration: number;
+  cancelled: boolean;
+  charactersCount: number;
+  streamed: boolean;
+  error?: Error;
+}
+
+export interface VADMetrics {
+  timestamp: number;
+  idleTime: number;
+  inferenceDurationTotal: number;
+  inferenceCount: number;
+  label: string;
+}
+
+export interface PipelineEOUMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+  /** Timestamp of when the event was recorded */
+  timestamp: number;
+  /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
+  endOfUtteranceDelay: number;
+  /**
+   * Time taken to obtain the transcript after the end of the user's speech.
+   *
+   * @remarks
+   * May be 0 if the transcript was already available.
+   */
+  transcriptionDelay: number;
+}
+
+export interface PipelineLLMMetrics extends LLMMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export interface PipelineTTSMetrics extends TTSMetrics {
+  /**
+   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   */
+  sequenceId: string;
+}
+
+export type PipelineSTTMetrics = STTMetrics;
+export type PipelineVADMetrics = VADMetrics;
+
+export class MultimodalLLMError extends Error {
+  type?: string;
+  reason?: string;
+  code?: string;
+  constructor(
+    {
+      type,
+      reason,
+      code,
+      message,
+    }: { type?: string; reason?: string; code?: string; message?: string } = {},
+    options?: ErrorOptions,
+  ) {
+    super(message, options);
+    this.type = type;
+    this.reason = reason;
+    this.code = code;
+  }
+}
+
+export interface MultimodalLLMMetrics extends LLMMetrics {
+  inputTokenDetails: {
+    cachedTokens: number;
+    textTokens: number;
+    audioTokens: number;
+  };
+  outputTokenDetails: {
+    textTokens: number;
+    audioTokens: number;
+  };
+}
+
+export type AgentMetrics =
+  | STTMetrics
+  | LLMMetrics
+  | TTSMetrics
+  | VADMetrics
+  | PipelineSTTMetrics
+  | PipelineEOUMetrics
+  | PipelineLLMMetrics
+  | PipelineTTSMetrics
+  | PipelineVADMetrics
+  | MultimodalLLMMetrics;

package/src/metrics/index.ts

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type {
+  AgentMetrics,
+  STTMetrics,
+  LLMMetrics,
+  TTSMetrics,
+  VADMetrics,
+  PipelineSTTMetrics,
+  PipelineEOUMetrics,
+  PipelineLLMMetrics,
+  PipelineTTSMetrics,
+  PipelineVADMetrics,
+  MultimodalLLMMetrics,
+} from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';

package/src/metrics/usage_collector.ts

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AgentMetrics } from './base.js';
+import { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';
+
+export interface UsageSummary {
+  llmPromptTokens: number;
+  llmCompletionTokens: number;
+  ttsCharactersCount: number;
+  sttAudioDuration: number;
+}
+
+export class UsageCollector {
+  #summary: UsageSummary;
+
+  constructor() {
+    this.#summary = {
+      llmPromptTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDuration: 0,
+    };
+  }
+
+  collect(metrics: AgentMetrics) {
+    if (isLLMMetrics(metrics)) {
+      this.#summary.llmPromptTokens += metrics.promptTokens;
+      this.#summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (isTTSMetrics(metrics)) {
+      this.#summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (isSTTMetrics(metrics)) {
+      this.#summary.sttAudioDuration += metrics.audioDuration;
+    }
+  }
+
+  get summary(): UsageSummary {
+    return { ...this.#summary };
+  }
+}

package/src/metrics/utils.ts

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../log.js';
+import type {
+  AgentMetrics,
+  LLMMetrics,
+  PipelineEOUMetrics,
+  PipelineLLMMetrics,
+  PipelineTTSMetrics,
+  STTMetrics,
+  TTSMetrics,
+  VADMetrics,
+} from './base.js';
+
+export const logMetrics = (metrics: AgentMetrics) => {
+  const logger = log();
+  if (isPipelineLLMMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        ttft: metrics.ttft,
+        inputTokens: metrics.promptTokens,
+        outputTokens: metrics.completionTokens,
+        tokensPerSecond: metrics.tokensPerSecond,
+      })
+      .info('Pipeline LLM metrics');
+  } else if (isLLMMetrics(metrics)) {
+    logger
+      .child({
+        ttft: metrics.ttft,
+        inputTokens: metrics.promptTokens,
+        outputTokens: metrics.completionTokens,
+        tokensPerSecond: metrics.tokensPerSecond,
+      })
+      .info('LLM metrics');
+  } else if (isPipelineTTSMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        ttfb: metrics.ttfb,
+        audioDuration: metrics.audioDuration,
+      })
+      .info('Pipeline TTS metrics');
+  } else if (isTTSMetrics(metrics)) {
+    logger
+      .child({
+        ttfb: metrics.ttfb,
+        audioDuration: metrics.audioDuration,
+      })
+      .info('TTS metrics');
+  } else if (isPipelineEOUMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        endOfUtteranceDelay: metrics.endOfUtteranceDelay,
+        transcriptionDelay: metrics.transcriptionDelay,
+      })
+      .info('Pipeline EOU metrics');
+  } else if (isSTTMetrics(metrics)) {
+    logger
+      .child({
+        audioDuration: metrics.audioDuration,
+      })
+      .info('STT metrics');
+  }
+};
+
+export const isLLMMetrics = (metrics: AgentMetrics): metrics is LLMMetrics => {
+  return !!(metrics as LLMMetrics).ttft;
+};
+
+export const isPipelineLLMMetrics = (metrics: AgentMetrics): metrics is PipelineLLMMetrics => {
+  return isLLMMetrics(metrics) && !!(metrics as PipelineLLMMetrics).sequenceId;
+};
+
+export const isVADMetrics = (metrics: AgentMetrics): metrics is VADMetrics => {
+  return !!(metrics as VADMetrics).inferenceCount;
+};
+
+export const isPipelineEOUMetrics = (metrics: AgentMetrics): metrics is PipelineEOUMetrics => {
+  return !!(metrics as PipelineEOUMetrics).endOfUtteranceDelay;
+};
+
+export const isTTSMetrics = (metrics: AgentMetrics): metrics is TTSMetrics => {
+  return !!(metrics as TTSMetrics).ttfb;
+};
+
+export const isPipelineTTSMetrics = (metrics: AgentMetrics): metrics is PipelineTTSMetrics => {
+  return isTTSMetrics(metrics) && !!(metrics as PipelineTTSMetrics).sequenceId;
+};
+
+export const isSTTMetrics = (metrics: AgentMetrics): metrics is STTMetrics => {
+  return !(
+    isLLMMetrics(metrics) ||
+    isVADMetrics(metrics) ||
+    isPipelineEOUMetrics(metrics) ||
+    isTTSMetrics(metrics)
+  );
+};

package/src/multimodal/multimodal_agent.ts

@@ -21,6 +21,7 @@ import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
 import * as llm from '../llm/index.js';
 import { log } from '../log.js';
+import type { MultimodalLLMMetrics } from '../metrics/base.js';
 import { BasicTranscriptionForwarder } from '../transcription.js';
 import { findMicroTrackId } from '../utils.js';
 import { AgentPlayout, type PlayoutHandle } from './agent_playout.js';
@@ -35,6 +36,7 @@ export abstract class RealtimeSession extends EventEmitter {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer
   abstract fncCtx: llm.FunctionContext | undefined;
+  abstract recoverFromTextResponse(itemId: string): void;
 }
 
 /**
@@ -60,21 +62,27 @@ export class MultimodalAgent extends EventEmitter {
   room: Room | null = null;
   linkedParticipant: RemoteParticipant | null = null;
   subscribedTrack: RemoteAudioTrack | null = null;
-  readMicroTask: { promise: Promise<void>; cancel: () => void } | null = null;
+  readMicroTask: Promise<void> | null = null;
+
+  #textResponseRetries = 0;
+  #maxTextResponseRetries: number;
 
   constructor({
     model,
     chatCtx,
     fncCtx,
+    maxTextResponseRetries = 5,
   }: {
     model: RealtimeModel;
     chatCtx?: llm.ChatContext;
     fncCtx?: llm.FunctionContext;
+    maxTextResponseRetries?: number;
   }) {
     super();
     this.model = model;
     this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
+    this.#maxTextResponseRetries = maxTextResponseRetries;
   }
 
   #participant: RemoteParticipant | string | null = null;
@@ -145,7 +153,7 @@ export class MultimodalAgent extends EventEmitter {
         if (this.linkedParticipant) {
           return;
         }
-        this.#linkParticipant(participant.identity);
+        this.#linkParticipant(participant.identity!);
       });
       room.on(
         RoomEvent.TrackPublished,
@@ -219,12 +227,12 @@ export class MultimodalAgent extends EventEmitter {
         if (typeof participant === 'string') {
           this.#linkParticipant(participant);
         } else {
-          this.#linkParticipant(participant.identity);
+          this.#linkParticipant(participant.identity!);
         }
       } else {
         // No participant specified, try to find the first participant in the room
         for (const participant of room.remoteParticipants.values()) {
-          this.#linkParticipant(participant.identity);
+          this.#linkParticipant(participant.identity!);
           break;
         }
       }
@@ -235,9 +243,11 @@ export class MultimodalAgent extends EventEmitter {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       this.#session.on('response_content_added', (message: any) => {
         // openai.realtime.RealtimeContent
+        if (message.contentType === 'text') return;
+
         const trFwd = new BasicTranscriptionForwarder(
           this.room!,
-          this.room!.localParticipant!.identity,
+          this.room!.localParticipant!.identity!,
           this.#getLocalTrackSid()!,
           message.responseId,
         );
@@ -252,6 +262,36 @@ export class MultimodalAgent extends EventEmitter {
         this.#playingHandle = handle;
       });
 
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('response_content_done', (message: any) => {
+        // openai.realtime.RealtimeContent
+        if (message.contentType === 'text') {
+          if (this.#textResponseRetries >= this.#maxTextResponseRetries) {
+            throw new Error(
+              'The OpenAI Realtime API returned a text response ' +
+                `after ${this.#maxTextResponseRetries} retries. ` +
+                'Please try to reduce the number of text system or ' +
+                'assistant messages in the chat context.',
+            );
+          }
+
+          this.#textResponseRetries++;
+          this.#logger
+            .child({
+              itemId: message.itemId,
+              text: message.text,
+              retries: this.#textResponseRetries,
+            })
+            .warn(
+              'The OpenAI Realtime API returned a text response instead of audio. ' +
+                'Attempting to recover to audio mode...',
+            );
+          this.#session!.recoverFromTextResponse(message.itemId);
+        } else {
+          this.#textResponseRetries = 0;
+        }
+      });
+
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       this.#session.on('input_speech_committed', (ev: any) => {
         // openai.realtime.InputSpeechCommittedEvent
@@ -284,6 +324,7 @@ export class MultimodalAgent extends EventEmitter {
       });
 
       this.#session.on('input_speech_started', (ev: any) => {
+        this.emit('user_started_speaking');
         if (this.#playingHandle && !this.#playingHandle.done) {
           this.#playingHandle.interrupt();
 
@@ -326,6 +367,10 @@ export class MultimodalAgent extends EventEmitter {
         this.#updateState();
       });
 
+      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {
+        this.emit('metrics_collected', metrics);
+      });
+
       resolve(this.#session);
     });
   }
@@ -404,27 +449,16 @@ export class MultimodalAgent extends EventEmitter {
     };
     this.subscribedTrack = track;
 
-    if (this.readMicroTask) {
-      this.readMicroTask.cancel();
-    }
-
-    let cancel: () => void;
-    this.readMicroTask = {
-      promise: new Promise<void>((resolve, reject) => {
-        cancel = () => {
-          reject(new Error('Task cancelled'));
-        };
-        readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
-          .then(resolve)
-          .catch(reject);
-      }),
-      cancel: () => cancel(),
-    };
+    this.readMicroTask = new Promise<void>((resolve, reject) => {
+      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
+        .then(resolve)
+        .catch(reject);
+    });
   }
 
   #getLocalTrackSid(): string | null {
     if (!this.#localTrackSid && this.room && this.room.localParticipant) {
-      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);
+      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant!.identity!);
     }
     return this.#localTrackSid;
   }
@@ -475,7 +509,7 @@ export class MultimodalAgent extends EventEmitter {
 
   #setState(state: AgentState) {
     if (this.room?.isConnected && this.room.localParticipant) {
-      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
+      const currentState = this.room.localParticipant.attributes![AGENT_STATE_ATTRIBUTE];
       if (currentState !== state) {
         this.room.localParticipant.setAttributes({
           [AGENT_STATE_ATTRIBUTE]: state,

package/src/pipeline/index.ts

@@ -7,9 +7,9 @@ export {
   type BeforeTTSCallback,
   type BeforeLLMCallback,
   type VPACallbacks,
-  type AgentCallContext,
   type AgentTranscriptionOptions,
   type VPAOptions,
   VPAEvent,
   VoicePipelineAgent,
+  AgentCallContext,
 } from './pipeline_agent.js';