@livekit/agents 0.5.2 → 0.6.1

package/src/pipeline/pipeline_agent.ts

@@ -17,10 +17,11 @@ import type {
   FunctionContext,
   LLM,
 } from '../llm/index.js';
-import { LLMStream } from '../llm/index.js';
+import { LLMEvent, LLMStream } from '../llm/index.js';
 import { ChatContext, ChatMessage, ChatRole } from '../llm/index.js';
 import { log } from '../log.js';
-import { type STT, StreamAdapter as STTStreamAdapter } from '../stt/index.js';
+import type { AgentMetrics, PipelineEOUMetrics } from '../metrics/base.js';
+import { type STT, StreamAdapter as STTStreamAdapter, SpeechEventType } from '../stt/index.js';
 import {
   SentenceTokenizer as BasicSentenceTokenizer,
   WordTokenizer as BasicWordTokenizer,
@@ -28,9 +29,9 @@ import {
 } from '../tokenize/basic/index.js';
 import type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';
 import type { TTS } from '../tts/index.js';
-import { StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
+import { TTSEvent, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
-import type { VAD, VADEvent } from '../vad.js';
+import { type VAD, type VADEvent, VADEventType } from '../vad.js';
 import type { SpeechSource, SynthesisHandle } from './agent_output.js';
 import { AgentOutput } from './agent_output.js';
 import { AgentPlayout, AgentPlayoutEvent } from './agent_playout.js';
@@ -39,6 +40,7 @@ import { SpeechHandle } from './speech_handle.js';
 
 export type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';
 export const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';
+let speechData: { sequenceId: string } | undefined;
 
 export type BeforeLLMCallback = (
   agent: VoicePipelineAgent,
@@ -60,6 +62,7 @@ export enum VPAEvent {
   AGENT_SPEECH_INTERRUPTED,
   FUNCTION_CALLS_COLLECTED,
   FUNCTION_CALLS_FINISHED,
+  METRICS_COLLECTED,
 }
 
 export type VPACallbacks = {
@@ -72,12 +75,14 @@ export type VPACallbacks = {
   [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;
   [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;
   [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;
+  [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;
 };
 
 export class AgentCallContext {
   #agent: VoicePipelineAgent;
   #llmStream: LLMStream;
   #metadata = new Map<string, any>();
+  #extraChatMessages: ChatMessage[] = [];
   static #current: AgentCallContext;
 
   constructor(agent: VoicePipelineAgent, llmStream: LLMStream) {
@@ -105,6 +110,14 @@ export class AgentCallContext {
   get llmStream(): LLMStream {
     return this.#llmStream;
   }
+
+  get extraChatMessages() {
+    return this.#extraChatMessages;
+  }
+
+  addExtraChatMessage(message: ChatMessage) {
+    this.#extraChatMessages.push(message);
+  }
 }
 
 const defaultBeforeLLMCallback: BeforeLLMCallback = (
@@ -171,7 +184,7 @@ export interface VPAOptions {
   interruptMinWords: number;
   /** Delay to wait before considering the user speech done. */
   minEndpointingDelay: number;
-  maxRecursiveFncCalls: number;
+  maxNestedFncCalls: number;
   /* Whether to preemptively synthesize responses. */
   preemptiveSynthesis: boolean;
   /*
@@ -201,7 +214,7 @@ const defaultVPAOptions: VPAOptions = {
   interruptSpeechDuration: 50,
   interruptMinWords: 0,
   minEndpointingDelay: 500,
-  maxRecursiveFncCalls: 1,
+  maxNestedFncCalls: 1,
   preemptiveSynthesis: false,
   beforeLLMCallback: defaultBeforeLLMCallback,
   beforeTTSCallback: defaultBeforeTTSCallback,
@@ -229,7 +242,6 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
   #transcribedInterimText = '';
   #speechQueueOpen = new Future();
   #speechQueue = new AsyncIterableQueue<SpeechHandle | typeof VoicePipelineAgent.FLUSH_SENTINEL>();
-  #lastEndOfSpeechTime?: number;
   #updateStateTask?: CancellablePromise<void>;
   #started = false;
   #room?: Room;
@@ -237,6 +249,8 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
   #deferredValidation: DeferredReplyValidation;
   #logger = log();
   #agentPublication?: LocalTrackPublication;
+  #lastFinalTranscriptTime?: number;
+  #lastSpeechTime?: number;
 
   constructor(
     /** Voice Activity Detection instance. */
@@ -317,12 +331,31 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     if (this.#started) {
       throw new Error('voice assistant already started');
     }
+
+    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(VPAEvent.METRICS_COLLECTED, metrics);
+    });
+
+    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {
+      if (!speechData) return;
+      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });
+    });
+
+    this.#llm.on(LLMEvent.METRICS_COLLECTED, (metrics) => {
+      if (!speechData) return;
+      this.emit(VPAEvent.METRICS_COLLECTED, { ...metrics, sequenceId: speechData.sequenceId });
+    });
+
+    this.#vad.on(VADEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(VPAEvent.METRICS_COLLECTED, metrics);
+    });
+
     room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {
       // automatically link to the first participant that connects, if not already linked
       if (this.#participant) {
         return;
       }
-      this.#linkParticipant.call(this, participant.identity);
+      this.#linkParticipant.call(this, participant.identity!);
     });
 
     this.#room = room;
@@ -332,7 +365,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       if (typeof participant === 'string') {
         this.#linkParticipant(participant);
       } else {
-        this.#linkParticipant(participant.identity);
+        this.#linkParticipant(participant.identity!);
       }
     }
 
@@ -344,12 +377,51 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     source: string | LLMStream | AsyncIterable<string>,
     allowInterruptions = true,
     addToChatCtx = true,
-  ) {
+  ): Promise<SpeechHandle> {
     await this.#trackPublishedFut.await;
+
+    let callContext: AgentCallContext | undefined;
+    let fncSource: string | AsyncIterable<string> | undefined;
+    if (addToChatCtx) {
+      callContext = AgentCallContext.getCurrent();
+      if (source instanceof LLMStream) {
+        this.#logger.warn('LLMStream will be ignored for function call chat context');
+      } else if (typeof source === 'string') {
+        fncSource = source;
+      } else {
+        fncSource = source;
+        source = new AsyncIterableQueue<string>();
+      }
+    }
+
     const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);
     const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);
     newHandle.initialize(source, synthesisHandle);
-    this.#addSpeechForPlayout(newHandle);
+
+    if (this.#playingSpeech && !this.#playingSpeech.nestedSpeechFinished) {
+      this.#playingSpeech.addNestedSpeech(newHandle);
+    } else {
+      this.#addSpeechForPlayout(newHandle);
+    }
+
+    if (callContext && fncSource) {
+      let text: string;
+      if (typeof source === 'string') {
+        text = fncSource as string;
+      } else {
+        text = '';
+        for await (const chunk of fncSource) {
+          (source as AsyncIterableQueue<string>).put(chunk);
+          text += chunk;
+        }
+        (source as AsyncIterableQueue<string>).close();
+      }
+
+      callContext.addExtraChatMessage(ChatMessage.create({ text, role: ChatRole.ASSISTANT }));
+      this.#logger.child({ text }).debug('added speech to function call chat context');
+    }
+
+    return newHandle;
   }
 
   #updateState(state: AgentState, delay = 0) {
@@ -410,11 +482,14 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       if (event.speechDuration >= this.#opts.interruptSpeechDuration) {
         this.#interruptIfPossible();
       }
+
+      if (event.rawAccumulatedSpeech > 0) {
+        this.#lastSpeechTime = Date.now() - event.rawAccumulatedSilence;
+      }
     });
     this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {
       this.emit(VPAEvent.USER_STARTED_SPEAKING);
       this.#deferredValidation.onHumanEndOfSpeech(event);
-      this.#lastEndOfSpeechTime = Date.now();
     });
     this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
       this.#transcribedInterimText = event.alternatives![0].text;
@@ -423,7 +498,7 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       const newTranscript = event.alternatives![0].text;
       if (!newTranscript) return;
 
-      this.#logger.child({ userTranscript: newTranscript }).debug('received user transcript');
+      this.#lastFinalTranscriptTime = Date.now();
       this.#transcribedText += (this.#transcribedText ? ' ' : '') + newTranscript;
 
       if (
@@ -534,33 +609,31 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
         }),
       );
 
-      if (cancelled) resolve();
-      let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
-      if (llmStream === false) {
-        handle?.cancel();
-        return;
-      }
-
-      if (cancelled) resolve();
-      // fallback to default impl if no custom/user stream is returned
-      if (!(llmStream instanceof LLMStream)) {
-        llmStream = (await defaultBeforeLLMCallback(this, copiedCtx)) as LLMStream;
-      }
+      speechData = { sequenceId: handle!.id };
 
-      if (handle!.interrupted) {
-        return;
-      }
+      try {
+        if (cancelled) resolve();
+        let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
+        if (llmStream === false) {
+          handle?.cancel();
+          return;
+        }
 
-      const synthesisHandle = this.#synthesizeAgentSpeech(handle!.id, llmStream);
-      handle!.initialize(llmStream, synthesisHandle);
+        if (cancelled) resolve();
+        // fallback to default impl if no custom/user stream is returned
+        if (!(llmStream instanceof LLMStream)) {
+          llmStream = (await defaultBeforeLLMCallback(this, copiedCtx)) as LLMStream;
+        }
 
-      // TODO(theomonnom): find a more reliable way to get the elapsed time from the last EOS
-      // (VAD could not have detected any speech — maybe unlikely?)
-      const elapsed = !!this.#lastEndOfSpeechTime
-        ? Math.round((Date.now() - this.#lastEndOfSpeechTime) * 1000) / 1000
-        : -1;
+        if (handle!.interrupted) {
+          return;
+        }
 
-      this.#logger.child({ speechId: handle!.id, elapsed }).debug('synthesizing agent reply');
+        const synthesisHandle = this.#synthesizeAgentSpeech(handle!.id, llmStream);
+        handle!.initialize(llmStream, synthesisHandle);
+      } finally {
+        speechData = undefined;
+      }
       resolve();
     });
   }
@@ -621,78 +694,107 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
 
     const collectedText = handle.synthesisHandle.text;
     const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
-    const extraToolsMessages = []; // additional messages from the functions to add to the context
-    let interrupted = handle.interrupted;
+    const interrupted = handle.interrupted;
+
+    const executeFunctionCalls = async () => {
+      // if the answer is using tools, execute the functions and automatically generate
+      // a response to the user question from the returned values
+      if (!isUsingTools || interrupted) return;
+
+      if (handle.fncNestedDepth >= this.#opts.maxNestedFncCalls) {
+        this.#logger
+          .child({ speechId: handle.id, fncNestedDepth: handle.fncNestedDepth })
+          .warn('max function calls nested depth reached');
+        return;
+      }
 
-    // if the answer is using tools, execute the functions and automatically generate
-    // a response to the user question from the returned values
-    if (isUsingTools && !interrupted) {
       if (!userQuestion || !handle.userCommitted) {
         throw new Error('user speech should have been committed before using tools');
       }
       const llmStream = handle.source;
-      let newFunctionCalls = llmStream.functionCalls;
-
-      for (let i = 0; i < this.#opts.maxRecursiveFncCalls; i++) {
-        this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);
-        const calledFuncs: FunctionCallInfo[] = [];
-        for (const func of newFunctionCalls) {
-          const task = func.func.execute(func.params).then(
-            (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),
-            (error) => ({ name: func.name, toolCallId: func.toolCallId, error }),
-          );
-          calledFuncs.push({ ...func, task });
+      const newFunctionCalls = llmStream.functionCalls;
+
+      new AgentCallContext(this, llmStream);
+
+      this.emit(VPAEvent.FUNCTION_CALLS_COLLECTED, newFunctionCalls);
+      const calledFuncs: FunctionCallInfo[] = [];
+      for (const func of newFunctionCalls) {
+        const task = func.func.execute(func.params).then(
+          (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),
+          (error) => ({ name: func.name, toolCallId: func.toolCallId, error }),
+        );
+        calledFuncs.push({ ...func, task });
+        this.#logger
+          .child({ function: func.name, speechId: handle.id })
+          .debug('executing AI function');
+        try {
+          await task;
+        } catch {
           this.#logger
             .child({ function: func.name, speechId: handle.id })
-            .debug('executing AI function');
-          try {
-            await task;
-          } catch {
-            this.#logger
-              .child({ function: func.name, speechId: handle.id })
-              .error('error executing AI function');
-          }
+            .error('error executing AI function');
         }
+      }
 
-        const toolCallsInfo = [];
-        const toolCallsResults = [];
-        for (const fnc of calledFuncs) {
-          // ignore the function calls that return void
-          const task = await fnc.task;
-          if (!task || task.result === undefined) continue;
-          toolCallsInfo.push(fnc);
-          toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));
-        }
+      const toolCallsInfo = [];
+      const toolCallsResults = [];
+      for (const fnc of calledFuncs) {
+        // ignore the function calls that return void
+        const task = await fnc.task;
+        if (!task || task.result === undefined) continue;
+        toolCallsInfo.push(fnc);
+        toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));
+      }
 
-        if (!toolCallsInfo.length) break;
+      if (!toolCallsInfo.length) return;
 
-        // generate an answer from the tool calls
-        extraToolsMessages.push(ChatMessage.createToolCalls(toolCallsInfo, collectedText));
-        extraToolsMessages.push(...toolCallsResults);
+      // generate an answer from the tool calls
+      const extraToolsMessages = [ChatMessage.createToolCalls(toolCallsInfo, collectedText)];
+      extraToolsMessages.push(...toolCallsResults);
 
-        const chatCtx = handle.source.chatCtx.copy();
-        chatCtx.messages.push(...extraToolsMessages);
+      // create a nested speech handle
+      const newSpeechHandle = SpeechHandle.createToolSpeech(
+        handle.allowInterruptions,
+        handle.addToChatCtx,
+        handle.fncNestedDepth + 1,
+        extraToolsMessages,
+      );
 
-        const answerLLMStream = this.llm.chat({
-          chatCtx,
-          fncCtx: this.fncCtx,
-        });
-        const answerSynthesis = this.#synthesizeAgentSpeech(handle.id, answerLLMStream);
-        // replace the synthesis handle with the new one to allow interruption
-        handle.synthesisHandle = answerSynthesis;
-        const playHandle = answerSynthesis.play();
-        await playHandle.join().await;
+      // synthesize the tool speech with the chat ctx from llmStream
+      const chatCtx = handle.source.chatCtx.copy();
+      chatCtx.messages.push(...extraToolsMessages);
+      chatCtx.messages.push(...AgentCallContext.getCurrent().extraChatMessages);
 
-        interrupted = answerSynthesis.interrupted;
-        newFunctionCalls = answerLLMStream.functionCalls;
+      const answerLLMStream = this.llm.chat({
+        chatCtx,
+        fncCtx: this.fncCtx,
+      });
+      const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);
+      newSpeechHandle.initialize(answerLLMStream, answerSynthesis);
+      handle.addNestedSpeech(newSpeechHandle);
+
+      this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);
+    };
 
-        this.emit(VPAEvent.FUNCTION_CALLS_FINISHED, calledFuncs);
-        if (!newFunctionCalls) break;
+    const task = executeFunctionCalls().then(() => {
+      handle.markNestedSpeechFinished();
+    });
+    while (!handle.nestedSpeechFinished) {
+      const changed = handle.nestedSpeechChanged();
+      await Promise.race([changed, task]);
+      while (handle.nestedSpeechHandles.length) {
+        const speech = handle.nestedSpeechHandles[0]!;
+        this.#playingSpeech = speech;
+        await this.#playSpeech(speech);
+        handle.nestedSpeechHandles.shift();
+        this.#playingSpeech = handle;
       }
     }
 
     if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
-      this.chatCtx.messages.push(...extraToolsMessages);
+      if (handle.extraToolsMessages) {
+        this.chatCtx.messages.push(...handle.extraToolsMessages);
+      }
       if (interrupted) {
         collectedText + '…';
       }
@@ -714,6 +816,8 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
           speechId: handle.id,
         })
         .debug('committed agent speech');
+
+      handle.setDone();
     }
   }
 
@@ -773,6 +877,21 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
 
     this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug('validated agent reply');
 
+    if (this.#lastSpeechTime) {
+      const timeSinceLastSpeech = Date.now() - this.#lastSpeechTime;
+      const transcriptionDelay = Math.max(
+        (this.#lastFinalTranscriptTime || 0) - this.#lastSpeechTime,
+        0,
+      );
+      const metrics: PipelineEOUMetrics = {
+        timestamp: Date.now(),
+        sequenceId: this.#pendingAgentReply.id,
+        endOfUtteranceDelay: timeSinceLastSpeech,
+        transcriptionDelay,
+      };
+      this.emit(VPAEvent.METRICS_COLLECTED, metrics);
+    }
+
     this.#addSpeechForPlayout(this.#pendingAgentReply);
     this.#pendingAgentReply = undefined;
     this.#transcribedInterimText = '';

package/src/pipeline/speech_handle.ts

@@ -2,8 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { randomUUID } from 'crypto';
-import type { LLMStream } from '../llm/index.js';
-import { Future } from '../utils.js';
+import type { ChatMessage, LLMStream } from '../llm/index.js';
+import { AsyncIterableQueue, Future } from '../utils.js';
 import type { SynthesisHandle } from './agent_output.js';
 
 export class SpeechHandle {
@@ -14,10 +14,16 @@ export class SpeechHandle {
   #userQuestion: string;
   #userCommitted = false;
   #initFut = new Future();
+  #doneFut = new Future();
   #speechCommitted = false;
   #source?: string | LLMStream | AsyncIterable<string>;
   #synthesisHandle?: SynthesisHandle;
   #initialized = false;
+  #fncNestedDepth: number;
+  #fncExtraToolsMesages?: ChatMessage[];
+  #nestedSpeechHandles: SpeechHandle[] = [];
+  #nestedSpeechChanged = new AsyncIterableQueue<void>();
+  #nestedSpeechFinished = false;
 
   constructor(
     id: string,
@@ -25,12 +31,16 @@ export class SpeechHandle {
     addToChatCtx: boolean,
     isReply: boolean,
     userQuestion: string,
+    fncNestedDepth = 0,
+    extraToolsMessages: ChatMessage[] | undefined = undefined,
   ) {
     this.#id = id;
     this.#allowInterruptions = allowInterruptions;
     this.#addToChatCtx = addToChatCtx;
     this.#isReply = isReply;
     this.#userQuestion = userQuestion;
+    this.#fncNestedDepth = fncNestedDepth;
+    this.#fncExtraToolsMesages = extraToolsMessages;
   }
 
   static createAssistantReply(
@@ -45,6 +55,23 @@ export class SpeechHandle {
     return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, false, '');
   }
 
+  static createToolSpeech(
+    allowInterruptions: boolean,
+    addToChatCtx: boolean,
+    fncNestedDepth: number,
+    extraToolsMessages: ChatMessage[],
+  ): SpeechHandle {
+    return new SpeechHandle(
+      randomUUID(),
+      allowInterruptions,
+      addToChatCtx,
+      false,
+      '',
+      fncNestedDepth,
+      extraToolsMessages,
+    );
+  }
+
   async waitForInitialization() {
     await this.#initFut.await;
   }
@@ -122,6 +149,43 @@ export class SpeechHandle {
     return !!this.#synthesisHandle?.interrupted;
   }
 
+  get fncNestedDepth(): number {
+    return this.#fncNestedDepth;
+  }
+
+  get extraToolsMessages(): ChatMessage[] | undefined {
+    return this.#fncExtraToolsMesages;
+  }
+
+  addNestedSpeech(handle: SpeechHandle) {
+    this.#nestedSpeechHandles.push(handle);
+    this.#nestedSpeechChanged.put();
+  }
+
+  get nestedSpeechHandles(): SpeechHandle[] {
+    return this.#nestedSpeechHandles;
+  }
+
+  async nestedSpeechChanged() {
+    await this.#nestedSpeechChanged.next();
+  }
+
+  get nestedSpeechFinished(): boolean {
+    return this.#nestedSpeechFinished;
+  }
+
+  markNestedSpeechFinished() {
+    this.#nestedSpeechFinished = true;
+  }
+
+  join() {
+    return this.#doneFut.await;
+  }
+
+  setDone() {
+    this.#doneFut.resolve();
+  }
+
   interrupt() {
     if (!this.#allowInterruptions) {
       throw new Error('interruptions are not allowed');
@@ -131,6 +195,7 @@ export class SpeechHandle {
 
   cancel() {
     this.#initFut.reject(new Error());
+    this.#nestedSpeechChanged.close();
     this.#synthesisHandle?.interrupt();
   }
 }

package/src/stt/index.ts

@@ -6,6 +6,8 @@ export {
   type SpeechEvent,
   type SpeechData,
   type STTCapabilities,
+  type RecognitionUsage,
+  type STTCallbacks,
   SpeechEventType,
   STT,
   SpeechStream,

package/src/stt/stream_adapter.ts

@@ -10,14 +10,20 @@ import { STT, SpeechEventType, SpeechStream } from './stt.js';
 export class StreamAdapter extends STT {
   #stt: STT;
   #vad: VAD;
+  label: string;
 
   constructor(stt: STT, vad: VAD) {
     super({ streaming: true, interimResults: false });
     this.#stt = stt;
     this.#vad = vad;
+    this.label = `stt.StreamAdapter<${this.#stt.label}>`;
+
+    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);
+    });
   }
 
-  recognize(frame: AudioFrame): Promise<SpeechEvent> {
+  _recognize(frame: AudioFrame): Promise<SpeechEvent> {
     return this.#stt.recognize(frame);
   }
 
@@ -29,15 +35,21 @@ export class StreamAdapter extends STT {
 export class StreamAdapterWrapper extends SpeechStream {
   #stt: STT;
   #vadStream: VADStream;
+  label: string;
 
   constructor(stt: STT, vad: VAD) {
-    super();
+    super(stt);
     this.#stt = stt;
     this.#vadStream = vad.stream();
+    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;
 
     this.#run();
   }
 
+  async monitorMetrics() {
+    return; // do nothing
+  }
+
   async #run() {
     const forwardInput = async () => {
       for await (const input of this.input) {
@@ -54,17 +66,17 @@ export class StreamAdapterWrapper extends SpeechStream {
       for await (const ev of this.#vadStream) {
         switch (ev.type) {
           case VADEventType.START_OF_SPEECH:
-            this.queue.put({ type: SpeechEventType.START_OF_SPEECH });
+            this.output.put({ type: SpeechEventType.START_OF_SPEECH });
             break;
           case VADEventType.END_OF_SPEECH:
-            this.queue.put({ type: SpeechEventType.END_OF_SPEECH });
+            this.output.put({ type: SpeechEventType.END_OF_SPEECH });
 
             const event = await this.#stt.recognize(ev.frames);
             if (!event.alternatives![0].text) {
               continue;
             }
 
-            this.queue.put(event);
+            this.output.put(event);
             break;
         }
       }