@livekit/agents 0.5.1 → 0.6.0

package/dist/pipeline/pipeline_agent.d.ts

@@ -3,11 +3,13 @@ import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { CallableFunctionResult, FunctionCallInfo, FunctionContext, LLM } from '../llm/index.js';
 import { LLMStream } from '../llm/index.js';
 import { ChatContext, ChatMessage } from '../llm/index.js';
+import type { AgentMetrics } from '../metrics/base.js';
 import { type STT } from '../stt/index.js';
 import type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';
 import type { TTS } from '../tts/index.js';
-import type { VAD } from '../vad.js';
+import { type VAD } from '../vad.js';
 import type { SpeechSource } from './agent_output.js';
+import { SpeechHandle } from './speech_handle.js';
 export type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';
 export declare const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
 export type BeforeLLMCallback = (agent: VoicePipelineAgent, chatCtx: ChatContext) => LLMStream | false | void | Promise<LLMStream | false | void>;
@@ -21,7 +23,8 @@ export declare enum VPAEvent {
     AGENT_SPEECH_COMMITTED = 5,
     AGENT_SPEECH_INTERRUPTED = 6,
     FUNCTION_CALLS_COLLECTED = 7,
-    FUNCTION_CALLS_FINISHED = 8
+    FUNCTION_CALLS_FINISHED = 8,
+    METRICS_COLLECTED = 9
 }
 export type VPACallbacks = {
     [VPAEvent.USER_STARTED_SPEAKING]: () => void;
@@ -33,6 +36,7 @@ export type VPACallbacks = {
     [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;
     [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;
     [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;
+    [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;
 };
 export declare class AgentCallContext {
     #private;
@@ -42,6 +46,8 @@ export declare class AgentCallContext {
     storeMetadata(key: string, value: any): void;
     getMetadata(key: string, orDefault?: any): any;
     get llmStream(): LLMStream;
+    get extraChatMessages(): ChatMessage[];
+    addExtraChatMessage(message: ChatMessage): void;
 }
 export interface AgentTranscriptionOptions {
     /** Whether to forward the user transcription to the client */
@@ -82,7 +88,7 @@ export interface VPAOptions {
     interruptMinWords: number;
     /** Delay to wait before considering the user speech done. */
     minEndpointingDelay: number;
-    maxRecursiveFncCalls: number;
+    maxNestedFncCalls: number;
     preemptiveSynthesis: boolean;
     beforeLLMCallback: BeforeLLMCallback;
     beforeTTSCallback: BeforeTTSCallback;
@@ -127,7 +133,7 @@ export declare class VoicePipelineAgent extends VoicePipelineAgent_base {
      */
     participant?: RemoteParticipant | string | null): void;
     /** Play a speech source through the voice assistant. */
-    say(source: string | LLMStream | AsyncIterable<string>, allowInterruptions?: boolean, addToChatCtx?: boolean): Promise<void>;
+    say(source: string | LLMStream | AsyncIterable<string>, allowInterruptions?: boolean, addToChatCtx?: boolean): Promise<SpeechHandle>;
     /** Close the voice assistant. */
     close(): Promise<void>;
 }

package/dist/pipeline/pipeline_agent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline_agent.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline_agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAyB,iBAAiB,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAQxF,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EACV,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,GAAG,EACJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAY,MAAM,iBAAiB,CAAC;AAErE,OAAO,EAAE,KAAK,GAAG,EAAqC,MAAM,iBAAiB,CAAC;AAM9E,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACjF,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAG3C,OAAO,KAAK,EAAE,GAAG,EAAY,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,YAAY,EAAmB,MAAM,mBAAmB,CAAC;AAMvE,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAEtD,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE,WAAW,KACjB,SAAS,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;AAElE,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KACnC,YAAY,CAAC;AAElB,oBAAY,QAAQ;IAClB,qBAAqB,IAAA;IACrB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,sBAAsB,IAAA;IACtB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,wBAAwB,IAAA;IACxB,wBAAwB,IAAA;IACxB,uBAAuB,IAAA;CACxB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7D,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC9D,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAChE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;IACzE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,IAAI,CAAC;CAC/E,CAAC;AAEF,qBAAa,gBAAgB;;gBAMf,KAAK,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS;IAM3D,MAAM,CAAC,UAAU,IAAI,gBAAgB;IAIrC,IAAI,KAAK,IAAI,kBAAkB,CAE9B;IAED,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG;IAIrC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,GAAe;IAInD,IAAI,SAAS,IAAI,SAAS,CAEzB;CACF;AAiBD,MAAM,WAAW,yBAAyB;IACxC,8DAA8D;IAC9D,iBAAiB,EAAE,OAAO,CAAC;IAC3B,+DAA+D;IAC/D,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,aAAa,EAAE,aAAa,CAAC;IAC7B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC3C;AAWD,MAAM,WAAW,UAAU;IACzB,sCAAsC;IACtC,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,4DAA4D;IAC5D,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+DAA+D;IAC/D,uBAAuB,EAAE,MAAM,CAAC;IAChC,sFAAsF;IACtF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6DAA6D;IAC7D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oBAAoB,EAAE,MAAM,CAAC;IAE7B,mBAAmB,EAAE,OAAO,CAAC;IAS7B,iBAAiB,EAAE,iBAAiB,CAAC;IAQrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,2CAA2C;IAC3C,aAAa,EAAE,yBAAyB,CAAC;CAC1C;iDAgBkE,aAAa,YAAY,CAAC;AAD7F,+DAA+D;AAC/D,qBAAa,kBAAmB,SAAQ,uBAAsD;;IAC5F,mFAAmF;IACnF,QAAQ,CAAC,0BAA0B,OAAO;IAC1C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;;IA2BlE,yCAAyC;IACzC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,qCAAqC;IACrC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,6CAA6C;IAC7C,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAyB/C,IAAI,MAAM,IAAI,eAAe,GAAG,SAAS,CAExC;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,eAAe,EAE9B;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,iCAAiC;IACjC,KAAK;IACH,8BAA8B;IAC9B,IAAI,EAAE,IAAI;IACV;;;;;;OAMG;IACH,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW;IA2BvD,wDAAwD;IAClD,GAAG,CACP,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,EAClD,kBAAkB,UAAO,EACzB,YAAY,UAAO;IAmdrB,iCAAiC;IAC3B,KAAK;CAQZ"}
+{"version":3,"file":"pipeline_agent.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline_agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAyB,iBAAiB,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAQxF,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EACV,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,GAAG,EACJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAY,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAY,MAAM,iBAAiB,CAAC;AAErE,OAAO,KAAK,EAAE,YAAY,EAAsB,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,KAAK,GAAG,EAAsD,MAAM,iBAAiB,CAAC;AAM/F,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACjF,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAG3C,OAAO,EAAE,KAAK,GAAG,EAA+B,MAAM,WAAW,CAAC;AAClE,OAAO,KAAK,EAAE,YAAY,EAAmB,MAAM,mBAAmB,CAAC;AAIvE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAGtD,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE,WAAW,KACjB,SAAS,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;AAElE,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KACnC,YAAY,CAAC;AAElB,oBAAY,QAAQ;IAClB,qBAAqB,IAAA;IACrB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,sBAAsB,IAAA;IACtB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,wBAAwB,IAAA;IACxB,wBAAwB,IAAA;IACxB,uBAAuB,IAAA;IACvB,iBAAiB,IAAA;CAClB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7D,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC9D,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAChE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;IACzE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,IAAI,CAAC;IAC9E,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/D,CAAC;AAEF,qBAAa,gBAAgB;;gBAOf,KAAK,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS;IAM3D,MAAM,CAAC,UAAU,IAAI,gBAAgB;IAIrC,IAAI,KAAK,IAAI,kBAAkB,CAE9B;IAED,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG;IAIrC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,GAAe;IAInD,IAAI,SAAS,IAAI,SAAS,CAEzB;IAED,IAAI,iBAAiB,kBAEpB;IAED,mBAAmB,CAAC,OAAO,EAAE,WAAW;CAGzC;AAiBD,MAAM,WAAW,yBAAyB;IACxC,8DAA8D;IAC9D,iBAAiB,EAAE,OAAO,CAAC;IAC3B,+DAA+D;IAC/D,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,aAAa,EAAE,aAAa,CAAC;IAC7B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC3C;AAWD,MAAM,WAAW,UAAU;IACzB,sCAAsC;IACtC,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,4DAA4D;IAC5D,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+DAA+D;IAC/D,uBAAuB,EAAE,MAAM,CAAC;IAChC,sFAAsF;IACtF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6DAA6D;IAC7D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAE1B,mBAAmB,EAAE,OAAO,CAAC;IAS7B,iBAAiB,EAAE,iBAAiB,CAAC;IAQrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,2CAA2C;IAC3C,aAAa,EAAE,yBAAyB,CAAC;CAC1C;iDAgBkE,aAAa,YAAY,CAAC;AAD7F,+DAA+D;AAC/D,qBAAa,kBAAmB,SAAQ,uBAAsD;;IAC5F,mFAAmF;IACnF,QAAQ,CAAC,0BAA0B,OAAO;IAC1C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;;IA4BlE,yCAAyC;IACzC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,qCAAqC;IACrC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,6CAA6C;IAC7C,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAyB/C,IAAI,MAAM,IAAI,eAAe,GAAG,SAAS,CAExC;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,eAAe,EAE9B;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,iCAAiC;IACjC,KAAK;IACH,8BAA8B;IAC9B,IAAI,EAAE,IAAI;IACV;;;;;;OAMG;IACH,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW;IA8CvD,wDAAwD;IAClD,GAAG,CACP,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,EAClD,kBAAkB,UAAO,EACzB,YAAY,UAAO,GAClB,OAAO,CAAC,YAAY,CAAC;IAoiBxB,iCAAiC;IAC3B,KAAK;CAQZ"}

package/dist/pipeline/pipeline_agent.js

@@ -6,22 +6,24 @@ import {
   TrackSource
 } from "@livekit/rtc-node";
 import EventEmitter from "node:events";
-import { LLMStream } from "../llm/index.js";
+import { LLMEvent, LLMStream } from "../llm/index.js";
 import { ChatContext, ChatMessage, ChatRole } from "../llm/index.js";
 import { log } from "../log.js";
-import { StreamAdapter as STTStreamAdapter } from "../stt/index.js";
+import { StreamAdapter as STTStreamAdapter, SpeechEventType } from "../stt/index.js";
 import {
   SentenceTokenizer as BasicSentenceTokenizer,
   WordTokenizer as BasicWordTokenizer,
   hyphenateWord
 } from "../tokenize/basic/index.js";
-import { StreamAdapter as TTSStreamAdapter } from "../tts/index.js";
+import { TTSEvent, StreamAdapter as TTSStreamAdapter } from "../tts/index.js";
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from "../utils.js";
+import { VADEventType } from "../vad.js";
 import { AgentOutput } from "./agent_output.js";
 import { AgentPlayout, AgentPlayoutEvent } from "./agent_playout.js";
 import { HumanInput, HumanInputEvent } from "./human_input.js";
 import { SpeechHandle } from "./speech_handle.js";
 const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
+let speechData;
 var VPAEvent = /* @__PURE__ */ ((VPAEvent2) => {
   VPAEvent2[VPAEvent2["USER_STARTED_SPEAKING"] = 0] = "USER_STARTED_SPEAKING";
   VPAEvent2[VPAEvent2["USER_STOPPED_SPEAKING"] = 1] = "USER_STOPPED_SPEAKING";
@@ -32,12 +34,14 @@ var VPAEvent = /* @__PURE__ */ ((VPAEvent2) => {
   VPAEvent2[VPAEvent2["AGENT_SPEECH_INTERRUPTED"] = 6] = "AGENT_SPEECH_INTERRUPTED";
   VPAEvent2[VPAEvent2["FUNCTION_CALLS_COLLECTED"] = 7] = "FUNCTION_CALLS_COLLECTED";
   VPAEvent2[VPAEvent2["FUNCTION_CALLS_FINISHED"] = 8] = "FUNCTION_CALLS_FINISHED";
+  VPAEvent2[VPAEvent2["METRICS_COLLECTED"] = 9] = "METRICS_COLLECTED";
   return VPAEvent2;
 })(VPAEvent || {});
 class AgentCallContext {
   #agent;
   #llmStream;
   #metadata = /* @__PURE__ */ new Map();
+  #extraChatMessages = [];
   static #current;
   constructor(agent, llmStream) {
     this.#agent = agent;
@@ -59,6 +63,12 @@ class AgentCallContext {
   get llmStream() {
     return this.#llmStream;
   }
+  get extraChatMessages() {
+    return this.#extraChatMessages;
+  }
+  addExtraChatMessage(message) {
+    this.#extraChatMessages.push(message);
+  }
 }
 const defaultBeforeLLMCallback = (agent, chatCtx) => {
   return agent.llm.chat({ chatCtx, fncCtx: agent.fncCtx });
@@ -80,7 +90,7 @@ const defaultVPAOptions = {
   interruptSpeechDuration: 50,
   interruptMinWords: 0,
   minEndpointingDelay: 500,
-  maxRecursiveFncCalls: 1,
+  maxNestedFncCalls: 1,
   preemptiveSynthesis: false,
   beforeLLMCallback: defaultBeforeLLMCallback,
   beforeTTSCallback: defaultBeforeTTSCallback,
@@ -105,7 +115,6 @@ class VoicePipelineAgent extends EventEmitter {
   #transcribedInterimText = "";
   #speechQueueOpen = new Future();
   #speechQueue = new AsyncIterableQueue();
-  #lastEndOfSpeechTime;
   #updateStateTask;
   #started = false;
   #room;
@@ -113,6 +122,8 @@ class VoicePipelineAgent extends EventEmitter {
   #deferredValidation;
   #logger = log();
   #agentPublication;
+  #lastFinalTranscriptTime;
+  #lastSpeechTime;
   constructor(vad, stt, llm, tts, opts = defaultVPAOptions) {
     super();
     this.#opts = { ...defaultVPAOptions, ...opts };
@@ -157,6 +168,20 @@ class VoicePipelineAgent extends EventEmitter {
     if (this.#started) {
       throw new Error("voice assistant already started");
     }
+    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(9 /* METRICS_COLLECTED */, metrics);
+    });
+    this.#tts.on(TTSEvent.METRICS_COLLECTED, (metrics) => {
+      if (!speechData) return;
+      this.emit(9 /* METRICS_COLLECTED */, { ...metrics, sequenceId: speechData.sequenceId });
+    });
+    this.#llm.on(LLMEvent.METRICS_COLLECTED, (metrics) => {
+      if (!speechData) return;
+      this.emit(9 /* METRICS_COLLECTED */, { ...metrics, sequenceId: speechData.sequenceId });
+    });
+    this.#vad.on(VADEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(9 /* METRICS_COLLECTED */, metrics);
+    });
     room.on(RoomEvent.ParticipantConnected, (participant2) => {
       if (this.#participant) {
         return;
@@ -177,10 +202,43 @@ class VoicePipelineAgent extends EventEmitter {
   /** Play a speech source through the voice assistant. */
   async say(source, allowInterruptions = true, addToChatCtx = true) {
     await this.#trackPublishedFut.await;
+    let callContext;
+    let fncSource;
+    if (addToChatCtx) {
+      callContext = AgentCallContext.getCurrent();
+      if (source instanceof LLMStream) {
+        this.#logger.warn("LLMStream will be ignored for function call chat context");
+      } else if (typeof source === "string") {
+        fncSource = source;
+      } else {
+        fncSource = source;
+        source = new AsyncIterableQueue();
+      }
+    }
     const newHandle = SpeechHandle.createAssistantSpeech(allowInterruptions, addToChatCtx);
     const synthesisHandle = this.#synthesizeAgentSpeech(newHandle.id, source);
     newHandle.initialize(source, synthesisHandle);
-    this.#addSpeechForPlayout(newHandle);
+    if (this.#playingSpeech && !this.#playingSpeech.nestedSpeechFinished) {
+      this.#playingSpeech.addNestedSpeech(newHandle);
+    } else {
+      this.#addSpeechForPlayout(newHandle);
+    }
+    if (callContext && fncSource) {
+      let text;
+      if (typeof source === "string") {
+        text = fncSource;
+      } else {
+        text = "";
+        for await (const chunk of fncSource) {
+          source.put(chunk);
+          text += chunk;
+        }
+        source.close();
+      }
+      callContext.addExtraChatMessage(ChatMessage.create({ text, role: ChatRole.ASSISTANT }));
+      this.#logger.child({ text }).debug("added speech to function call chat context");
+    }
+    return newHandle;
   }
   #updateState(state, delay = 0) {
     const runTask = (delay2) => {
@@ -234,11 +292,13 @@ class VoicePipelineAgent extends EventEmitter {
       if (event.speechDuration >= this.#opts.interruptSpeechDuration) {
         this.#interruptIfPossible();
       }
+      if (event.rawAccumulatedSpeech > 0) {
+        this.#lastSpeechTime = Date.now() - event.rawAccumulatedSilence;
+      }
     });
     this.#humanInput.on(HumanInputEvent.END_OF_SPEECH, (event) => {
       this.emit(0 /* USER_STARTED_SPEAKING */);
       this.#deferredValidation.onHumanEndOfSpeech(event);
-      this.#lastEndOfSpeechTime = Date.now();
     });
     this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
       this.#transcribedInterimText = event.alternatives[0].text;
@@ -246,7 +306,7 @@ class VoicePipelineAgent extends EventEmitter {
     this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
       const newTranscript = event.alternatives[0].text;
       if (!newTranscript) return;
-      this.#logger.child({ userTranscript: newTranscript }).debug("received user transcript");
+      this.#lastFinalTranscriptTime = Date.now();
       this.#transcribedText += (this.#transcribedText ? " " : "") + newTranscript;
       if (this.#opts.preemptiveSynthesis && (!this.#playingSpeech || this.#playingSpeech.allowInterruptions)) {
         this.#synthesizeAgentReply();
@@ -318,8 +378,7 @@ class VoicePipelineAgent extends EventEmitter {
         if ((!playingSpeech.userQuestion || playingSpeech.userCommitted) && !playingSpeech.speechCommitted) {
           copiedCtx.messages.push(
             ChatMessage.create({
-              // TODO(nbsp): uhhh unsure where to get the played text here
-              // text: playingSpeech.synthesisHandle.(theres no ttsForwarder here)
+              text: playingSpeech.synthesisHandle.text,
               role: ChatRole.ASSISTANT
             })
           );
@@ -331,23 +390,26 @@ class VoicePipelineAgent extends EventEmitter {
           role: ChatRole.USER
         })
       );
-      if (cancelled) resolve();
-      let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
-      if (llmStream === false) {
-        handle == null ? void 0 : handle.cancel();
-        return;
-      }
-      if (cancelled) resolve();
-      if (!(llmStream instanceof LLMStream)) {
-        llmStream = await defaultBeforeLLMCallback(this, copiedCtx);
-      }
-      if (handle.interrupted) {
-        return;
+      speechData = { sequenceId: handle.id };
+      try {
+        if (cancelled) resolve();
+        let llmStream = await this.#opts.beforeLLMCallback(this, copiedCtx);
+        if (llmStream === false) {
+          handle == null ? void 0 : handle.cancel();
+          return;
+        }
+        if (cancelled) resolve();
+        if (!(llmStream instanceof LLMStream)) {
+          llmStream = await defaultBeforeLLMCallback(this, copiedCtx);
+        }
+        if (handle.interrupted) {
+          return;
+        }
+        const synthesisHandle = this.#synthesizeAgentSpeech(handle.id, llmStream);
+        handle.initialize(llmStream, synthesisHandle);
+      } finally {
+        speechData = void 0;
       }
-      const synthesisHandle = this.#synthesizeAgentSpeech(handle.id, llmStream);
-      handle.initialize(llmStream, synthesisHandle);
-      const elapsed = !!this.#lastEndOfSpeechTime ? Math.round((Date.now() - this.#lastEndOfSpeechTime) * 1e3) / 1e3 : -1;
-      this.#logger.child({ speechId: handle.id, elapsed }).debug("synthesizing agent reply");
       resolve();
     });
   }
@@ -387,62 +449,83 @@ class VoicePipelineAgent extends EventEmitter {
       if (handle.interrupted) break;
     }
     commitUserQuestionIfNeeded();
-    let collectedText = "";
+    const collectedText = handle.synthesisHandle.text;
     const isUsingTools = handle.source instanceof LLMStream && !!handle.source.functionCalls.length;
-    const extraToolsMessages = [];
-    let interrupted = handle.interrupted;
-    if (isUsingTools && !interrupted) {
+    const interrupted = handle.interrupted;
+    const executeFunctionCalls = async () => {
+      if (!isUsingTools || interrupted) return;
+      if (handle.fncNestedDepth >= this.#opts.maxNestedFncCalls) {
+        this.#logger.child({ speechId: handle.id, fncNestedDepth: handle.fncNestedDepth }).warn("max function calls nested depth reached");
+        return;
+      }
       if (!userQuestion || !handle.userCommitted) {
         throw new Error("user speech should have been committed before using tools");
       }
       const llmStream = handle.source;
-      let newFunctionCalls = llmStream.functionCalls;
-      for (let i = 0; i < this.#opts.maxRecursiveFncCalls; i++) {
-        this.emit(7 /* FUNCTION_CALLS_COLLECTED */, newFunctionCalls);
-        const calledFuncs = [];
-        for (const func of newFunctionCalls) {
-          const task = func.func.execute(func.params).then(
-            (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),
-            (error) => ({ name: func.name, toolCallId: func.toolCallId, error })
-          );
-          calledFuncs.push({ ...func, task });
-          this.#logger.child({ function: func.name, speechId: handle.id }).debug("executing AI function");
-          try {
-            await task;
-          } catch {
-            this.#logger.child({ function: func.name, speechId: handle.id }).error("error executing AI function");
-          }
-        }
-        const toolCallsInfo = [];
-        const toolCallsResults = [];
-        for (const fnc of calledFuncs) {
-          const task = await fnc.task;
-          if (!task || task.result === void 0) continue;
-          toolCallsInfo.push(fnc);
-          toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task));
+      const newFunctionCalls = llmStream.functionCalls;
+      new AgentCallContext(this, llmStream);
+      this.emit(7 /* FUNCTION_CALLS_COLLECTED */, newFunctionCalls);
+      const calledFuncs = [];
+      for (const func of newFunctionCalls) {
+        const task2 = func.func.execute(func.params).then(
+          (result) => ({ name: func.name, toolCallId: func.toolCallId, result }),
+          (error) => ({ name: func.name, toolCallId: func.toolCallId, error })
+        );
+        calledFuncs.push({ ...func, task: task2 });
+        this.#logger.child({ function: func.name, speechId: handle.id }).debug("executing AI function");
+        try {
+          await task2;
+        } catch {
+          this.#logger.child({ function: func.name, speechId: handle.id }).error("error executing AI function");
         }
-        if (!toolCallsInfo.length) break;
-        extraToolsMessages.push(ChatMessage.createToolCalls(toolCallsInfo, collectedText));
-        extraToolsMessages.push(...toolCallsResults);
-        const chatCtx = handle.source.chatCtx.copy();
-        chatCtx.messages.push(...extraToolsMessages);
-        const answerLLMStream = this.llm.chat({
-          chatCtx,
-          fncCtx: this.fncCtx
-        });
-        const answerSynthesis = this.#synthesizeAgentSpeech(handle.id, answerLLMStream);
-        handle.synthesisHandle = answerSynthesis;
-        const playHandle2 = answerSynthesis.play();
-        await playHandle2.join().await;
-        collectedText = "";
-        interrupted = answerSynthesis.interrupted;
-        newFunctionCalls = answerLLMStream.functionCalls;
-        this.emit(8 /* FUNCTION_CALLS_FINISHED */, calledFuncs);
-        if (!newFunctionCalls) break;
+      }
+      const toolCallsInfo = [];
+      const toolCallsResults = [];
+      for (const fnc of calledFuncs) {
+        const task2 = await fnc.task;
+        if (!task2 || task2.result === void 0) continue;
+        toolCallsInfo.push(fnc);
+        toolCallsResults.push(ChatMessage.createToolFromFunctionResult(task2));
+      }
+      if (!toolCallsInfo.length) return;
+      const extraToolsMessages = [ChatMessage.createToolCalls(toolCallsInfo, collectedText)];
+      extraToolsMessages.push(...toolCallsResults);
+      const newSpeechHandle = SpeechHandle.createToolSpeech(
+        handle.allowInterruptions,
+        handle.addToChatCtx,
+        handle.fncNestedDepth + 1,
+        extraToolsMessages
+      );
+      const chatCtx = handle.source.chatCtx.copy();
+      chatCtx.messages.push(...extraToolsMessages);
+      chatCtx.messages.push(...AgentCallContext.getCurrent().extraChatMessages);
+      const answerLLMStream = this.llm.chat({
+        chatCtx,
+        fncCtx: this.fncCtx
+      });
+      const answerSynthesis = this.#synthesizeAgentSpeech(newSpeechHandle.id, answerLLMStream);
+      newSpeechHandle.initialize(answerLLMStream, answerSynthesis);
+      handle.addNestedSpeech(newSpeechHandle);
+      this.emit(8 /* FUNCTION_CALLS_FINISHED */, calledFuncs);
+    };
+    const task = executeFunctionCalls().then(() => {
+      handle.markNestedSpeechFinished();
+    });
+    while (!handle.nestedSpeechFinished) {
+      const changed = handle.nestedSpeechChanged();
+      await Promise.race([changed, task]);
+      while (handle.nestedSpeechHandles.length) {
+        const speech = handle.nestedSpeechHandles[0];
+        this.#playingSpeech = speech;
+        await this.#playSpeech(speech);
+        handle.nestedSpeechHandles.shift();
+        this.#playingSpeech = handle;
       }
     }
     if (handle.addToChatCtx && (!userQuestion || handle.userCommitted)) {
-      this.chatCtx.messages.push(...extraToolsMessages);
+      if (handle.extraToolsMessages) {
+        this.chatCtx.messages.push(...handle.extraToolsMessages);
+      }
       if (interrupted) {
         collectedText + "\u2026";
       }
@@ -459,6 +542,7 @@ class VoicePipelineAgent extends EventEmitter {
         interrupted,
         speechId: handle.id
       }).debug("committed agent speech");
+      handle.setDone();
     }
   }
   #synthesizeAgentSpeech(speechId, source) {
@@ -499,6 +583,20 @@ class VoicePipelineAgent extends EventEmitter {
       }
     }
     this.#logger.child({ speechId: this.#pendingAgentReply.id }).debug("validated agent reply");
+    if (this.#lastSpeechTime) {
+      const timeSinceLastSpeech = Date.now() - this.#lastSpeechTime;
+      const transcriptionDelay = Math.max(
+        (this.#lastFinalTranscriptTime || 0) - this.#lastSpeechTime,
+        0
+      );
+      const metrics = {
+        timestamp: Date.now(),
+        sequenceId: this.#pendingAgentReply.id,
+        endOfUtteranceDelay: timeSinceLastSpeech,
+        transcriptionDelay
+      };
+      this.emit(9 /* METRICS_COLLECTED */, metrics);
+    }
     this.#addSpeechForPlayout(this.#pendingAgentReply);
     this.#pendingAgentReply = void 0;
     this.#transcribedInterimText = "";