@livekit/agents 1.0.22 → 1.0.24

package/src/voice/audio_recognition.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { AudioFrame } from '@livekit/rtc-node';
+import type { Context, Span } from '@opentelemetry/api';
 import type { WritableStreamDefaultWriter } from 'node:stream/web';
 import { ReadableStream } from 'node:stream/web';
 import { type ChatContext } from '../llm/chat_context.js';
@@ -10,6 +11,7 @@ import { DeferredReadableStream, isStreamReaderReleaseError } from '../stream/de
 import { IdentityTransform } from '../stream/identity_transform.js';
 import { mergeReadableStreams } from '../stream/merge_readable_streams.js';
 import { type SpeechEvent, SpeechEventType } from '../stt/stt.js';
+import { traceTypes, tracer } from '../telemetry/index.js';
 import { Task, delay } from '../utils.js';
 import { type VAD, type VADEvent, VADEventType } from '../vad.js';
 import type { TurnDetectionMode } from './agent_session.js';
@@ -55,10 +57,9 @@ export interface AudioRecognitionOptions {
   turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
   minEndpointingDelay: number;
   maxEndpointingDelay: number;
+  rootSpanContext?: Context;
 }
 
-// TODO(brian): PR3 - Add span: private _userTurnSpan?: Span, create lazily in _ensureUserTurnSpan() method (tracer.startSpan('user_turn') with participant attributes)
-// TODO(brian): PR3 - Add span: 'eou_detection' span when running EOU detection (in runEOUDetection method)
 export class AudioRecognition {
   private hooks: RecognitionHooks;
   private stt?: STTNode;
@@ -68,6 +69,7 @@ export class AudioRecognition {
   private minEndpointingDelay: number;
   private maxEndpointingDelay: number;
   private lastLanguage?: string;
+  private rootSpanContext?: Context;
 
   private deferredInputStream: DeferredReadableStream<AudioFrame>;
   private logger = log();
@@ -82,6 +84,8 @@ export class AudioRecognition {
   private speaking = false;
   private sampleRate?: number;
 
+  private userTurnSpan?: Span;
+
   private vadInputStream: ReadableStream<AudioFrame>;
   private sttInputStream: ReadableStream<AudioFrame>;
   private silenceAudioTransform = new IdentityTransform<AudioFrame>();
@@ -102,6 +106,7 @@ export class AudioRecognition {
     this.minEndpointingDelay = opts.minEndpointingDelay;
     this.maxEndpointingDelay = opts.maxEndpointingDelay;
     this.lastLanguage = undefined;
+    this.rootSpanContext = opts.rootSpanContext;
 
     this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
     const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
@@ -357,31 +362,47 @@ export class AudioRecognition {
         let endpointingDelay = this.minEndpointingDelay;
 
         if (turnDetector) {
-          this.logger.debug('Running turn detector model');
-          if (!(await turnDetector.supportsLanguage(this.lastLanguage))) {
-            this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
-          } else {
-            const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);
-            this.logger.debug(
-              { endOfTurnProbability, language: this.lastLanguage },
-              'end of turn probability',
-            );
-
-            const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);
-            this.logger.debug(
-              {
-                unlikelyThreshold,
-                endOfTurnProbability,
-                language: this.lastLanguage,
-                transcript: this.audioTranscript,
-              },
-              'EOU Detection',
-            );
+          await tracer.startActiveSpan(
+            async (span) => {
+              this.logger.debug('Running turn detector model');
+
+              let endOfTurnProbability = 0.0;
+              let unlikelyThreshold: number | undefined;
+
+              if (!(await turnDetector.supportsLanguage(this.lastLanguage))) {
+                this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
+              } else {
+                try {
+                  endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);
+                  unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);
+
+                  this.logger.debug(
+                    { endOfTurnProbability, unlikelyThreshold, language: this.lastLanguage },
+                    'end of turn probability',
+                  );
+
+                  if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {
+                    endpointingDelay = this.maxEndpointingDelay;
+                  }
+                } catch (error) {
+                  this.logger.error(error, 'Error predicting end of turn');
+                }
+              }
 
-            if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {
-              endpointingDelay = this.maxEndpointingDelay;
-            }
-          }
+              span.setAttribute(
+                traceTypes.ATTR_CHAT_CTX,
+                JSON.stringify(chatCtx.toJSON({ excludeTimestamp: false })),
+              );
+              span.setAttribute(traceTypes.ATTR_EOU_PROBABILITY, endOfTurnProbability);
+              span.setAttribute(traceTypes.ATTR_EOU_UNLIKELY_THRESHOLD, unlikelyThreshold ?? 0);
+              span.setAttribute(traceTypes.ATTR_EOU_DELAY, endpointingDelay);
+              span.setAttribute(traceTypes.ATTR_EOU_LANGUAGE, this.lastLanguage ?? '');
+            },
+            {
+              name: 'eou_detection',
+              context: this.rootSpanContext,
+            },
+          );
         }
 
         let extraSleep = endpointingDelay;
@@ -430,6 +451,13 @@ export class AudioRecognition {
         });
 
         if (committed) {
+          this._endUserTurnSpan({
+            transcript: this.audioTranscript,
+            confidence: confidenceAvg,
+            transcriptionDelay: transcriptionDelay ?? 0,
+            endOfUtteranceDelay: endOfUtteranceDelay ?? 0,
+          });
+
           // clear the transcript if the user turn was committed
           this.audioTranscript = '';
           this.finalTranscriptConfidence = [];
@@ -537,6 +565,13 @@ export class AudioRecognition {
             this.hooks.onStartOfSpeech(ev);
             this.speaking = true;
 
+            if (!this.userTurnSpan) {
+              this.userTurnSpan = tracer.startSpan({
+                name: 'user_turn',
+                context: this.rootSpanContext,
+              });
+            }
+
             // Capture sample rate from the first VAD event if not already set
             if (ev.frames.length > 0 && ev.frames[0]) {
               this.sampleRate = ev.frames[0].sampleRate;
@@ -646,12 +681,36 @@ export class AudioRecognition {
 
   async close() {
     this.detachInputAudioStream();
+    this.silenceAudioWriter.releaseLock();
     await this.commitUserTurnTask?.cancelAndWait();
     await this.sttTask?.cancelAndWait();
     await this.vadTask?.cancelAndWait();
     await this.bounceEOUTask?.cancelAndWait();
   }
 
+  private _endUserTurnSpan({
+    transcript,
+    confidence,
+    transcriptionDelay,
+    endOfUtteranceDelay,
+  }: {
+    transcript: string;
+    confidence: number;
+    transcriptionDelay: number;
+    endOfUtteranceDelay: number;
+  }): void {
+    if (this.userTurnSpan) {
+      this.userTurnSpan.setAttributes({
+        [traceTypes.ATTR_USER_TRANSCRIPT]: transcript,
+        [traceTypes.ATTR_TRANSCRIPT_CONFIDENCE]: confidence,
+        [traceTypes.ATTR_TRANSCRIPTION_DELAY]: transcriptionDelay,
+        [traceTypes.ATTR_END_OF_TURN_DELAY]: endOfUtteranceDelay,
+      });
+      this.userTurnSpan.end();
+      this.userTurnSpan = undefined;
+    }
+  }
+
   private get vadBaseTurnDetection() {
     return ['vad', undefined].includes(this.turnDetectionMode);
   }

package/src/voice/generation.ts

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import { AudioResampler } from '@livekit/rtc-node';
+import type { Span } from '@opentelemetry/api';
+import { context as otelContext } from '@opentelemetry/api';
 import type { ReadableStream, ReadableStreamDefaultReader } from 'stream/web';
 import {
   type ChatContext,
@@ -21,6 +23,7 @@ import {
 import { isZodSchema, parseZodSchema } from '../llm/zod-utils.js';
 import { log } from '../log.js';
 import { IdentityTransform } from '../stream/identity_transform.js';
+import { traceTypes, tracer } from '../telemetry/index.js';
 import { Future, Task, shortuuid, toError } from '../utils.js';
 import { type Agent, type ModelSettings, asyncLocalStorage, isStopResponse } from './agent.js';
 import type { AgentSession } from './agent_session.js';
@@ -377,7 +380,6 @@ export function updateInstructions(options: {
   }
 }
 
-// TODO(brian): PR3 - Add @tracer.startActiveSpan('llm_node') decorator/wrapper
 export function performLLMInference(
   node: LLMNode,
   chatCtx: ChatContext,
@@ -392,7 +394,13 @@ export function performLLMInference(
   const toolCallWriter = toolCallStream.writable.getWriter();
   const data = new _LLMGenerationData(textStream.readable, toolCallStream.readable);
 
-  const inferenceTask = async (signal: AbortSignal) => {
+  const _performLLMInferenceImpl = async (signal: AbortSignal, span: Span) => {
+    span.setAttribute(
+      traceTypes.ATTR_CHAT_CTX,
+      JSON.stringify(chatCtx.toJSON({ excludeTimestamp: false })),
+    );
+    span.setAttribute(traceTypes.ATTR_FUNCTION_TOOLS, JSON.stringify(Object.keys(toolCtx)));
+
     let llmStreamReader: ReadableStreamDefaultReader<string | ChatChunk> | null = null;
     let llmStream: ReadableStream<string | ChatChunk> | null = null;
 
@@ -448,6 +456,8 @@ export function performLLMInference(
         // No need to check if chunk is of type other than ChatChunk or string like in
         // Python since chunk is defined in the type ChatChunk | string in TypeScript
       }
+
+      span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, data.generatedText);
     } catch (error) {
       if (error instanceof DOMException && error.name === 'AbortError') {
         // Abort signal was triggered, handle gracefully
@@ -462,13 +472,21 @@ export function performLLMInference(
     }
   };
 
+  // Capture the current context (agent_turn) to ensure llm_node is properly parented
+  const currentContext = otelContext.active();
+
+  const inferenceTask = async (signal: AbortSignal) =>
+    tracer.startActiveSpan(async (span) => _performLLMInferenceImpl(signal, span), {
+      name: 'llm_node',
+      context: currentContext,
+    });
+
   return [
     Task.from((controller) => inferenceTask(controller.signal), controller, 'performLLMInference'),
     data,
   ];
 }
 
-// TODO(brian): PR3 - Add @tracer.startActiveSpan('tts_node') decorator/wrapper
 export function performTTSInference(
   node: TTSNode,
   text: ReadableStream<string>,
@@ -479,7 +497,7 @@ export function performTTSInference(
   const outputWriter = audioStream.writable.getWriter();
   const audioOutputStream = audioStream.readable;
 
-  const inferenceTask = async (signal: AbortSignal) => {
+  const _performTTSInferenceImpl = async (signal: AbortSignal) => {
     let ttsStreamReader: ReadableStreamDefaultReader<AudioFrame> | null = null;
     let ttsStream: ReadableStream<AudioFrame> | null = null;
 
@@ -514,6 +532,15 @@ export function performTTSInference(
     }
   };
 
+  // Capture the current context (agent_turn) to ensure tts_node is properly parented
+  const currentContext = otelContext.active();
+
+  const inferenceTask = async (signal: AbortSignal) =>
+    tracer.startActiveSpan(async () => _performTTSInferenceImpl(signal), {
+      name: 'tts_node',
+      context: currentContext,
+    });
+
   return [
     Task.from((controller) => inferenceTask(controller.signal), controller, 'performTTSInference'),
     audioOutputStream,
@@ -652,7 +679,7 @@ export function performAudioForwarding(
   ];
 }
 
-// TODO(brian): PR3 - Add @tracer.startActiveSpan('function_tool') wrapper for each tool execution
+// function_tool span is already implemented in tracableToolExecution below (line ~796)
 export function performToolExecutions({
   session,
   speechHandle,
@@ -788,8 +815,9 @@ export function performToolExecutions({
         });
       });
 
-      const tracableToolExecution = async (toolExecTask: Promise<unknown>) => {
-        // TODO(brian): add tracing
+      const _tracableToolExecutionImpl = async (toolExecTask: Promise<unknown>, span: Span) => {
+        span.setAttribute(traceTypes.ATTR_FUNCTION_TOOL_NAME, toolCall.name);
+        span.setAttribute(traceTypes.ATTR_FUNCTION_TOOL_ARGS, toolCall.args);
 
         // await for task to complete, if task is aborted, set exception
         let toolOutput: ToolExecutionOutput | undefined;
@@ -800,6 +828,17 @@ export function performToolExecutions({
             exception: isAborted ? new Error('tool call was aborted') : undefined,
             output: isAborted ? undefined : result,
           });
+
+          if (toolOutput.toolCallOutput) {
+            span.setAttribute(
+              traceTypes.ATTR_FUNCTION_TOOL_OUTPUT,
+              toolOutput.toolCallOutput.output,
+            );
+            span.setAttribute(
+              traceTypes.ATTR_FUNCTION_TOOL_IS_ERROR,
+              toolOutput.toolCallOutput.isError,
+            );
+          }
         } catch (rawError) {
           logger.error(
             {
@@ -813,12 +852,25 @@ export function performToolExecutions({
             toolCall,
             exception: toError(rawError),
           });
+
+          if (toolOutput.toolCallOutput) {
+            span.setAttribute(
+              traceTypes.ATTR_FUNCTION_TOOL_OUTPUT,
+              toolOutput.toolCallOutput.output,
+            );
+            span.setAttribute(traceTypes.ATTR_FUNCTION_TOOL_IS_ERROR, true);
+          }
         } finally {
           if (!toolOutput) throw new Error('toolOutput is undefined');
           toolCompleted(toolOutput);
         }
       };
 
+      const tracableToolExecution = (toolExecTask: Promise<unknown>) =>
+        tracer.startActiveSpan(async (span) => _tracableToolExecutionImpl(toolExecTask, span), {
+          name: 'function_tool',
+        });
+
       // wait, not cancelling all tool calling tasks
       tasks.push(tracableToolExecution(toolExecution));
     }

package/src/voice/report.ts

@@ -12,7 +12,10 @@ export interface SessionReport {
   options: VoiceOptions;
   events: AgentEvent[];
   chatHistory: ChatContext;
-  enableUserDataTraining: boolean;
+  enableRecording: boolean;
+  /** Timestamp when the session started (milliseconds) */
+  startedAt: number;
+  /** Timestamp when the session report was created (milliseconds), typically at the end of the session */
   timestamp: number;
 }
 
@@ -24,6 +27,9 @@ export interface SessionReportOptions {
   events: AgentEvent[];
   chatHistory: ChatContext;
   enableUserDataTraining?: boolean;
+  /** Timestamp when the session started (milliseconds) */
+  startedAt?: number;
+  /** Timestamp when the session report was created (milliseconds) */
   timestamp?: number;
 }
 
@@ -35,12 +41,12 @@ export function createSessionReport(opts: SessionReportOptions): SessionReport {
     options: opts.options,
     events: opts.events,
     chatHistory: opts.chatHistory,
-    enableUserDataTraining: opts.enableUserDataTraining ?? false,
+    enableRecording: opts.enableUserDataTraining ?? false,
+    startedAt: opts.startedAt ?? Date.now(),
     timestamp: opts.timestamp ?? Date.now(),
   };
 }
 
-// TODO(brian): PR5 - Add uploadSessionReport() function that creates multipart form with:
 //   - header: protobuf MetricsRecordingHeader (room_id, duration, start_time)
 //   - chat_history: JSON serialized chat history (use sessionReportToJSON)
 //   - audio: audio recording file if available (ogg format)
@@ -71,7 +77,7 @@ export function sessionReportToJSON(report: SessionReport): Record<string, unkno
       max_tool_steps: report.options.maxToolSteps,
     },
     chat_history: report.chatHistory.toJSON({ excludeTimestamp: false }),
-    enable_user_data_training: report.enableUserDataTraining,
+    enable_user_data_training: report.enableRecording,
     timestamp: report.timestamp,
   };
 }