@livekit/agents 0.5.1 → 0.6.0

package/src/metrics/utils.ts

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../log.js';
+import type {
+  AgentMetrics,
+  LLMMetrics,
+  PipelineEOUMetrics,
+  PipelineLLMMetrics,
+  PipelineTTSMetrics,
+  STTMetrics,
+  TTSMetrics,
+  VADMetrics,
+} from './base.js';
+
+export const logMetrics = (metrics: AgentMetrics) => {
+  const logger = log();
+  if (isPipelineLLMMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        ttft: metrics.ttft,
+        inputTokens: metrics.promptTokens,
+        outputTokens: metrics.completionTokens,
+        tokensPerSecond: metrics.tokensPerSecond,
+      })
+      .info('Pipeline LLM metrics');
+  } else if (isLLMMetrics(metrics)) {
+    logger
+      .child({
+        ttft: metrics.ttft,
+        inputTokens: metrics.promptTokens,
+        outputTokens: metrics.completionTokens,
+        tokensPerSecond: metrics.tokensPerSecond,
+      })
+      .info('LLM metrics');
+  } else if (isPipelineTTSMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        ttfb: metrics.ttfb,
+        audioDuration: metrics.audioDuration,
+      })
+      .info('Pipeline TTS metrics');
+  } else if (isTTSMetrics(metrics)) {
+    logger
+      .child({
+        ttfb: metrics.ttfb,
+        audioDuration: metrics.audioDuration,
+      })
+      .info('TTS metrics');
+  } else if (isPipelineEOUMetrics(metrics)) {
+    logger
+      .child({
+        sequenceId: metrics.sequenceId,
+        endOfUtteranceDelay: metrics.endOfUtteranceDelay,
+        transcriptionDelay: metrics.transcriptionDelay,
+      })
+      .info('Pipeline EOU metrics');
+  } else if (isSTTMetrics(metrics)) {
+    logger
+      .child({
+        audioDuration: metrics.audioDuration,
+      })
+      .info('STT metrics');
+  }
+};
+
+export const isLLMMetrics = (metrics: AgentMetrics): metrics is LLMMetrics => {
+  return !!(metrics as LLMMetrics).ttft;
+};
+
+export const isPipelineLLMMetrics = (metrics: AgentMetrics): metrics is PipelineLLMMetrics => {
+  return isLLMMetrics(metrics) && !!(metrics as PipelineLLMMetrics).sequenceId;
+};
+
+export const isVADMetrics = (metrics: AgentMetrics): metrics is VADMetrics => {
+  return !!(metrics as VADMetrics).inferenceCount;
+};
+
+export const isPipelineEOUMetrics = (metrics: AgentMetrics): metrics is PipelineEOUMetrics => {
+  return !!(metrics as PipelineEOUMetrics).endOfUtteranceDelay;
+};
+
+export const isTTSMetrics = (metrics: AgentMetrics): metrics is TTSMetrics => {
+  return !!(metrics as TTSMetrics).ttfb;
+};
+
+export const isPipelineTTSMetrics = (metrics: AgentMetrics): metrics is PipelineTTSMetrics => {
+  return isTTSMetrics(metrics) && !!(metrics as PipelineTTSMetrics).sequenceId;
+};
+
+export const isSTTMetrics = (metrics: AgentMetrics): metrics is STTMetrics => {
+  return !(
+    isLLMMetrics(metrics) ||
+    isVADMetrics(metrics) ||
+    isPipelineEOUMetrics(metrics) ||
+    isTTSMetrics(metrics)
+  );
+};

package/src/multimodal/multimodal_agent.ts

@@ -21,6 +21,7 @@ import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
 import * as llm from '../llm/index.js';
 import { log } from '../log.js';
+import type { MultimodalLLMMetrics } from '../metrics/base.js';
 import { BasicTranscriptionForwarder } from '../transcription.js';
 import { findMicroTrackId } from '../utils.js';
 import { AgentPlayout, type PlayoutHandle } from './agent_playout.js';
@@ -60,7 +61,7 @@ export class MultimodalAgent extends EventEmitter {
   room: Room | null = null;
   linkedParticipant: RemoteParticipant | null = null;
   subscribedTrack: RemoteAudioTrack | null = null;
-  readMicroTask: { promise: Promise<void>; cancel: () => void } | null = null;
+  readMicroTask: Promise<void> | null = null;
 
   constructor({
     model,
@@ -284,6 +285,7 @@ export class MultimodalAgent extends EventEmitter {
       });
 
       this.#session.on('input_speech_started', (ev: any) => {
+        this.emit('user_started_speaking');
         if (this.#playingHandle && !this.#playingHandle.done) {
           this.#playingHandle.interrupt();
 
@@ -326,6 +328,10 @@ export class MultimodalAgent extends EventEmitter {
         this.#updateState();
       });
 
+      this.#session.on('metrics_collected', (metrics: MultimodalLLMMetrics) => {
+        this.emit('metrics_collected', metrics);
+      });
+
       resolve(this.#session);
     });
   }
@@ -404,22 +410,11 @@ export class MultimodalAgent extends EventEmitter {
     };
     this.subscribedTrack = track;
 
-    if (this.readMicroTask) {
-      this.readMicroTask.cancel();
-    }
-
-    let cancel: () => void;
-    this.readMicroTask = {
-      promise: new Promise<void>((resolve, reject) => {
-        cancel = () => {
-          reject(new Error('Task cancelled'));
-        };
-        readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
-          .then(resolve)
-          .catch(reject);
-      }),
-      cancel: () => cancel(),
-    };
+    this.readMicroTask = new Promise<void>((resolve, reject) => {
+      readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
+        .then(resolve)
+        .catch(reject);
+    });
   }
 
   #getLocalTrackSid(): string | null {

package/src/pipeline/agent_output.ts

@@ -13,6 +13,7 @@ export class SynthesisHandle {
   static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
 
   #speechId: string;
+  text?: string;
   ttsSource: SpeechSource;
   #agentPlayout: AgentPlayout;
   tts: TTS;
@@ -97,7 +98,7 @@ export class AgentOutput {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     return new CancellablePromise(async (resolve, _, onCancel) => {
       const ttsSource = await handle.ttsSource;
-      let task: CancellablePromise<void>;
+      let task: CancellablePromise<string>;
       if (typeof ttsSource === 'string') {
         task = stringSynthesisTask(ttsSource, handle);
       } else {
@@ -113,6 +114,10 @@ export class AgentOutput {
       } finally {
         if (handle.intFut.done) {
           gracefullyCancel(task);
+        } else {
+          task.then((text) => {
+            handle.text = text;
+          });
         }
       }
 
@@ -121,9 +126,9 @@ export class AgentOutput {
   }
 }
 
-const stringSynthesisTask = (text: string, handle: SynthesisHandle): CancellablePromise<void> => {
+const stringSynthesisTask = (text: string, handle: SynthesisHandle): CancellablePromise<string> => {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  return new CancellablePromise<void>(async (resolve, _, onCancel) => {
+  return new CancellablePromise(async (resolve, _, onCancel) => {
     let cancelled = false;
     onCancel(() => {
       cancelled = true;
@@ -141,16 +146,17 @@ const stringSynthesisTask = (text: string, handle: SynthesisHandle): Cancellable
     }
     handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
 
-    resolve();
+    resolve(text);
   });
 };
 
 const streamSynthesisTask = (
   stream: AsyncIterable<string>,
   handle: SynthesisHandle,
-): CancellablePromise<void> => {
+): CancellablePromise<string> => {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  return new CancellablePromise<void>(async (resolve, _, onCancel) => {
+  return new CancellablePromise(async (resolve, _, onCancel) => {
+    let fullText = '';
     let cancelled = false;
     onCancel(() => {
       cancelled = true;
@@ -170,12 +176,13 @@ const streamSynthesisTask = (
     readGeneratedAudio();
 
     for await (const text of stream) {
+      fullText += text;
       if (cancelled) break;
       ttsStream.pushText(text);
     }
     ttsStream.flush();
     ttsStream.endInput();
 
-    resolve();
+    resolve(fullText);
   });
 };

package/src/pipeline/index.ts

@@ -7,9 +7,9 @@ export {
   type BeforeTTSCallback,
   type BeforeLLMCallback,
   type VPACallbacks,
-  type AgentCallContext,
   type AgentTranscriptionOptions,
   type VPAOptions,
   VPAEvent,
   VoicePipelineAgent,
+  AgentCallContext,
 } from './pipeline_agent.js';