@livekit/agents 1.0.22 → 1.0.23

package/src/voice/agent_activity.ts

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 import { Mutex } from '@livekit/mutex';
 import type { AudioFrame } from '@livekit/rtc-node';
+import type { Span } from '@opentelemetry/api';
+import { ROOT_CONTEXT, trace } from '@opentelemetry/api';
 import { Heap } from 'heap-js';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { ReadableStream } from 'node:stream/web';
@@ -10,6 +12,7 @@ import { type ChatContext, ChatMessage } from '../llm/chat_context.js';
 import {
   type ChatItem,
   type FunctionCall,
+  type FunctionCallOutput,
   type GenerationCreatedEvent,
   type InputSpeechStartedEvent,
   type InputSpeechStoppedEvent,
@@ -34,6 +37,7 @@ import type {
 } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { STT, type STTError, type SpeechEvent } from '../stt/stt.js';
+import { traceTypes, tracer } from '../telemetry/index.js';
 import { splitWords } from '../tokenize/basic/word.js';
 import { TTS, type TTSError } from '../tts/tts.js';
 import { Future, Task, cancelAndWait, waitFor } from '../utils.js';
@@ -70,7 +74,6 @@ import {
 } from './generation.js';
 import { SpeechHandle } from './speech_handle.js';
 
-// equivalent to Python's contextvars
 const speechHandleStorage = new AsyncLocalStorage<SpeechHandle>();
 
 interface PreemptiveGeneration {
@@ -202,10 +205,15 @@ export class AgentActivity implements RecognitionHooks {
   }
 
   async start(): Promise<void> {
-    // TODO(brian): PR3 - Add span: startSpan = tracer.startSpan('start_agent_activity', { attributes: { 'lk.agent_label': this.agent.label } })
-    // TODO(brian): PR3 - Wrap prewarm calls with trace.useSpan(startSpan, endOnExit: false)
     const unlock = await this.lock.lock();
     try {
+      // Create start_agent_activity as a ROOT span (new trace) to match Python behavior
+      const startSpan = tracer.startSpan({
+        name: 'start_agent_activity',
+        attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+        context: ROOT_CONTEXT,
+      });
+
       this.agent._agentActivity = this;
 
       if (this.llm instanceof RealtimeModel) {
@@ -286,16 +294,26 @@ export class AgentActivity implements RecognitionHooks {
         turnDetectionMode: this.turnDetectionMode,
         minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
         maxEndpointingDelay: this.agentSession.options.maxEndpointingDelay,
+        rootSpanContext: this.agentSession.rootSpanContext,
       });
       this.audioRecognition.start();
       this.started = true;
 
       this._mainTask = Task.from(({ signal }) => this.mainTask(signal));
-      // TODO(brian): PR3 - Wrap onEnter with tracer.startActiveSpan('on_enter', { attributes: { 'lk.agent_label': this.agent.label }, context: startSpan context })
+
+      // Create on_enter as a child of start_agent_activity in the new trace
+      const onEnterTask = tracer.startActiveSpan(async () => this.agent.onEnter(), {
+        name: 'on_enter',
+        context: trace.setSpan(ROOT_CONTEXT, startSpan),
+        attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+      });
+
       this.createSpeechTask({
-        task: Task.from(() => this.agent.onEnter()),
+        task: Task.from(() => onEnterTask),
         name: 'AgentActivity_onEnter',
       });
+
+      startSpan.end();
     } finally {
       unlock();
     }
@@ -577,7 +595,6 @@ export class AgentActivity implements RecognitionHooks {
     }
 
     if (this.draining) {
-      // copied from python:
       // TODO(shubhra): should we "forward" this new turn to the next agent?
       this.logger.warn('skipping new realtime generation, the agent is draining');
       return;
@@ -783,7 +800,6 @@ export class AgentActivity implements RecognitionHooks {
     if (this.draining) {
       this.cancelPreemptiveGeneration();
       this.logger.warn({ user_input: info.newTranscript }, 'skipping user input, task is draining');
-      // copied from python:
       // TODO(shubhra): should we "forward" this new turn to the next agent/activity?
       return true;
     }
@@ -1254,17 +1270,35 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
-  // TODO(brian): PR3 - Wrap entire pipelineReplyTask() method with tracer.startActiveSpan('agent_turn')
-  private async pipelineReplyTask(
-    speechHandle: SpeechHandle,
-    chatCtx: ChatContext,
-    toolCtx: ToolContext,
-    modelSettings: ModelSettings,
-    replyAbortController: AbortController,
-    instructions?: string,
-    newMessage?: ChatMessage,
-    toolsMessages?: ChatItem[],
-  ): Promise<void> {
+  private _pipelineReplyTaskImpl = async ({
+    speechHandle,
+    chatCtx,
+    toolCtx,
+    modelSettings,
+    replyAbortController,
+    instructions,
+    newMessage,
+    toolsMessages,
+    span,
+  }: {
+    speechHandle: SpeechHandle;
+    chatCtx: ChatContext;
+    toolCtx: ToolContext;
+    modelSettings: ModelSettings;
+    replyAbortController: AbortController;
+    instructions?: string;
+    newMessage?: ChatMessage;
+    toolsMessages?: ChatItem[];
+    span: Span;
+  }): Promise<void> => {
+    span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
+    if (instructions) {
+      span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, instructions);
+    }
+    if (newMessage) {
+      span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || '');
+    }
+
     speechHandleStorage.enterWith(speechHandle);
 
     const audioOutput = this.agentSession.output.audioEnabled
@@ -1406,6 +1440,8 @@ export class AgentActivity implements RecognitionHooks {
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolsMessages);
+      // Also add to session history (matches Python agent_session.py _tool_items_added)
+      this.agentSession._toolItemsAdded(toolsMessages as (FunctionCall | FunctionCallOutput)[]);
     }
 
     if (speechHandle.interrupted) {
@@ -1601,8 +1637,38 @@ export class AgentActivity implements RecognitionHooks {
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolMessages);
+      this.agentSession._toolItemsAdded(toolMessages as (FunctionCall | FunctionCallOutput)[]);
     }
-  }
+  };
+
+  private pipelineReplyTask = async (
+    speechHandle: SpeechHandle,
+    chatCtx: ChatContext,
+    toolCtx: ToolContext,
+    modelSettings: ModelSettings,
+    replyAbortController: AbortController,
+    instructions?: string,
+    newMessage?: ChatMessage,
+    toolsMessages?: ChatItem[],
+  ): Promise<void> =>
+    tracer.startActiveSpan(
+      async (span) =>
+        this._pipelineReplyTaskImpl({
+          speechHandle,
+          chatCtx,
+          toolCtx,
+          modelSettings,
+          replyAbortController,
+          instructions,
+          newMessage,
+          toolsMessages,
+          span,
+        }),
+      {
+        name: 'agent_turn',
+        context: this.agentSession.rootSpanContext,
+      },
+    );
 
   private async realtimeGenerationTask(
     speechHandle: SpeechHandle,
@@ -1610,6 +1676,37 @@ export class AgentActivity implements RecognitionHooks {
     modelSettings: ModelSettings,
     replyAbortController: AbortController,
   ): Promise<void> {
+    return tracer.startActiveSpan(
+      async (span) =>
+        this._realtimeGenerationTaskImpl({
+          speechHandle,
+          ev,
+          modelSettings,
+          replyAbortController,
+          span,
+        }),
+      {
+        name: 'agent_turn',
+        context: this.agentSession.rootSpanContext,
+      },
+    );
+  }
+
+  private async _realtimeGenerationTaskImpl({
+    speechHandle,
+    ev,
+    modelSettings,
+    replyAbortController,
+    span,
+  }: {
+    speechHandle: SpeechHandle;
+    ev: GenerationCreatedEvent;
+    modelSettings: ModelSettings;
+    replyAbortController: AbortController;
+    span: Span;
+  }): Promise<void> {
+    span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
+
     speechHandleStorage.enterWith(speechHandle);
 
     if (!this.realtimeSession) {
@@ -1786,6 +1883,8 @@ export class AgentActivity implements RecognitionHooks {
 
     const onToolExecutionStarted = (f: FunctionCall) => {
       speechHandle._itemAdded([f]);
+      this.agent._chatCtx.items.push(f);
+      this.agentSession._toolItemsAdded([f]);
     };
 
     const onToolExecutionCompleted = (out: ToolExecutionOutput) => {
@@ -1979,6 +2078,11 @@ export class AgentActivity implements RecognitionHooks {
       }
       const chatCtx = this.realtimeSession.chatCtx.copy();
       chatCtx.items.push(...functionToolsExecutedEvent.functionCallOutputs);
+
+      this.agentSession._toolItemsAdded(
+        functionToolsExecutedEvent.functionCallOutputs as FunctionCallOutput[],
+      );
+
       try {
         await this.realtimeSession.updateChatCtx(chatCtx);
       } catch (error) {
@@ -2096,16 +2200,30 @@ export class AgentActivity implements RecognitionHooks {
     this.wakeupMainTask();
   }
 
-  // TODO(brian): PR3 - Wrap entire drain() method with tracer.startActiveSpan('drain_agent_activity', { attributes: { 'lk.agent_label': this.agent.label } })
   async drain(): Promise<void> {
+    // Create drain_agent_activity as a ROOT span (new trace) to match Python behavior
+    return tracer.startActiveSpan(async (span) => this._drainImpl(span), {
+      name: 'drain_agent_activity',
+      context: ROOT_CONTEXT,
+    });
+  }
+
+  private async _drainImpl(span: Span): Promise<void> {
+    span.setAttribute(traceTypes.ATTR_AGENT_LABEL, this.agent.id);
+
     const unlock = await this.lock.lock();
     try {
       if (this._draining) return;
 
       this.cancelPreemptiveGeneration();
-      // TODO(brian): PR3 - Wrap onExit with tracer.startActiveSpan('on_exit', { attributes: { 'lk.agent_label': this.agent.label } })
+
+      const onExitTask = tracer.startActiveSpan(async () => this.agent.onExit(), {
+        name: 'on_exit',
+        attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+      });
+
       this.createSpeechTask({
-        task: Task.from(() => this.agent.onExit()),
+        task: Task.from(() => onExitTask),
         name: 'AgentActivity_onExit',
       });
 

package/src/voice/agent_session.ts

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame, Room } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { Context, Span } from '@opentelemetry/api';
+import { ROOT_CONTEXT, context as otelContext, trace } from '@opentelemetry/api';
 import { EventEmitter } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
 import {
@@ -14,12 +16,14 @@ import {
   type TTSModelString,
 } from '../inference/index.js';
 import { getJobContext } from '../job.js';
+import type { FunctionCall, FunctionCallOutput } from '../llm/chat_context.js';
 import { AgentHandoffItem, ChatContext, ChatMessage } from '../llm/chat_context.js';
 import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import { log } from '../log.js';
 import type { STT } from '../stt/index.js';
 import type { STTError } from '../stt/stt.js';
+import { traceTypes, tracer } from '../telemetry/index.js';
 import type { TTS, TTSError } from '../tts/tts.js';
 import type { VAD } from '../vad.js';
 import type { Agent } from './agent.js';
@@ -128,9 +132,22 @@ export class AgentSession<
   private closingTask: Promise<void> | null = null;
   private userAwayTimer: NodeJS.Timeout | null = null;
 
+  private sessionSpan?: Span;
+  private userSpeakingSpan?: Span;
+  private agentSpeakingSpan?: Span;
+
+  /** @internal */
+  rootSpanContext?: Context;
+
   /** @internal */
   _recordedEvents: AgentEvent[] = [];
 
+  /** @internal */
+  _enableRecording = false;
+
+  /** @internal - Timestamp when the session started (milliseconds) */
+  _startedAt?: number;
+
   constructor(opts: AgentSessionOptions<UserData>) {
     super();
 
@@ -175,7 +192,8 @@ export class AgentSession<
     this._chatCtx = ChatContext.empty();
     this.options = { ...defaultVoiceOptions, ...voiceOptions };
 
-    this.on(AgentSessionEventTypes.UserInputTranscribed, this._onUserInputTranscribed.bind(this));
+    this._onUserInputTranscribed = this._onUserInputTranscribed.bind(this);
+    this.on(AgentSessionEventTypes.UserInputTranscribed, this._onUserInputTranscribed);
   }
 
   emit<K extends keyof AgentSessionCallbacks>(
@@ -211,25 +229,22 @@ export class AgentSession<
     this._userData = value;
   }
 
-  async start({
-    // TODO(brian): PR2 - Add setupCloudTracer() call if on LiveKit Cloud with recording enabled
-    // TODO(brian): PR3 - Add span: this._sessionSpan = tracer.startSpan('agent_session'), store as instance property
-    // TODO(brian): PR4 - Add setupCloudLogger() call in setupCloudTracer() to setup OTEL logging with Pino bridge
+  private async _startImpl({
     agent,
     room,
     inputOptions,
     outputOptions,
-    record = true,
+    record,
+    span,
   }: {
     agent: Agent;
     room: Room;
     inputOptions?: Partial<RoomInputOptions>;
     outputOptions?: Partial<RoomOutputOptions>;
-    record?: boolean;
+    record: boolean;
+    span: Span;
   }): Promise<void> {
-    if (this.started) {
-      return;
-    }
+    span.setAttribute(traceTypes.ATTR_AGENT_LABEL, agent.id);
 
     this.agent = agent;
     this._updateAgentState('initializing');
@@ -291,9 +306,62 @@ export class AgentSession<
     );
 
     this.started = true;
+    this._startedAt = Date.now();
     this._updateAgentState('listening');
   }
 
+  async start({
+    agent,
+    room,
+    inputOptions,
+    outputOptions,
+    record = true,
+  }: {
+    agent: Agent;
+    room: Room;
+    inputOptions?: Partial<RoomInputOptions>;
+    outputOptions?: Partial<RoomOutputOptions>;
+    record?: boolean;
+  }): Promise<void> {
+    if (this.started) {
+      return;
+    }
+
+    const ctx = getJobContext();
+
+    record = record ?? ctx.info.job.enableRecording;
+    this._enableRecording = record;
+
+    this.logger.info(
+      { record, enableRecording: ctx.info.job.enableRecording },
+      'Configuring session recording',
+    );
+
+    if (this._enableRecording) {
+      await ctx.initRecording();
+    }
+
+    // Create agent_session as a ROOT span (new trace) to match Python behavior
+    // This creates a separate trace for better cloud dashboard organization
+    this.sessionSpan = tracer.startSpan({
+      name: 'agent_session',
+      context: ROOT_CONTEXT,
+    });
+
+    // Set the session span as the active span in the context
+    // This ensures all child spans (agent_turn, user_turn, etc.) are parented to it
+    this.rootSpanContext = trace.setSpan(ROOT_CONTEXT, this.sessionSpan);
+
+    await this._startImpl({
+      agent,
+      room,
+      inputOptions,
+      outputOptions,
+      record,
+      span: this.sessionSpan,
+    });
+  }
+
   updateAgent(agent: Agent): void {
     this.agent = agent;
 
@@ -367,32 +435,41 @@ export class AgentSession<
   }
 
   private async updateActivity(agent: Agent): Promise<void> {
-    // TODO(AJS-129): add lock to agent activity core lifecycle
-    this.nextActivity = new AgentActivity(agent, this);
+    const runWithContext = async () => {
+      // TODO(AJS-129): add lock to agent activity core lifecycle
+      this.nextActivity = new AgentActivity(agent, this);
 
-    const previousActivity = this.activity;
+      const previousActivity = this.activity;
 
-    if (this.activity) {
-      await this.activity.drain();
-      await this.activity.close();
-    }
+      if (this.activity) {
+        await this.activity.drain();
+        await this.activity.close();
+      }
 
-    this.activity = this.nextActivity;
-    this.nextActivity = undefined;
+      this.activity = this.nextActivity;
+      this.nextActivity = undefined;
 
-    this._chatCtx.insert(
-      new AgentHandoffItem({
-        oldAgentId: previousActivity?.agent.id,
-        newAgentId: agent.id,
-      }),
-    );
-    this.logger.debug({ previousActivity, agent }, 'Agent handoff inserted into chat context');
+      this._chatCtx.insert(
+        new AgentHandoffItem({
+          oldAgentId: previousActivity?.agent.id,
+          newAgentId: agent.id,
+        }),
+      );
+      this.logger.debug({ previousActivity, agent }, 'Agent handoff inserted into chat context');
 
-    await this.activity.start();
+      await this.activity.start();
 
-    if (this._input.audio) {
-      this.activity.attachAudioInput(this._input.audio.stream);
+      if (this._input.audio) {
+        this.activity.attachAudioInput(this._input.audio.stream);
+      }
+    };
+
+    // Run within session span context if available
+    if (this.rootSpanContext) {
+      return otelContext.with(this.rootSpanContext, runWithContext);
     }
+
+    return runWithContext();
   }
 
   get chatCtx(): ChatContext {
@@ -452,14 +529,35 @@ export class AgentSession<
     this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));
   }
 
+  /** @internal */
+  _toolItemsAdded(items: (FunctionCall | FunctionCallOutput)[]): void {
+    this._chatCtx.insert(items);
+  }
+
   /** @internal */
   _updateAgentState(state: AgentState) {
     if (this._agentState === state) {
       return;
     }
 
-    // TODO(brian): PR3 - Add span: if state === 'speaking' && !this._agentSpeakingSpan, create tracer.startSpan('agent_speaking') with participant attributes
-    // TODO(brian): PR3 - Add span: if state !== 'speaking' && this._agentSpeakingSpan, end and clear this._agentSpeakingSpan
+    if (state === 'speaking') {
+      // TODO(brian): PR4 - Track error counts
+
+      if (this.agentSpeakingSpan === undefined) {
+        this.agentSpeakingSpan = tracer.startSpan({
+          name: 'agent_speaking',
+          context: this.rootSpanContext,
+        });
+
+        // TODO(brian): PR4 - Set participant attributes if roomIO.room.localParticipant is available
+        // (Ref: Python agent_session.py line 1161-1164)
+      }
+    } else if (this.agentSpeakingSpan !== undefined) {
+      // TODO(brian): PR4 - Set ATTR_END_TIME attribute if available
+      this.agentSpeakingSpan.end();
+      this.agentSpeakingSpan = undefined;
+    }
+
     const oldState = this._agentState;
     this._agentState = state;
 
@@ -482,8 +580,20 @@ export class AgentSession<
       return;
     }
 
-    // TODO(brian): PR3 - Add span: if state === 'speaking' && !this._userSpeakingSpan, create tracer.startSpan('user_speaking') with participant attributes
-    // TODO(brian): PR3 - Add span: if state !== 'speaking' && this._userSpeakingSpan, end and clear this._userSpeakingSpan
+    if (state === 'speaking' && this.userSpeakingSpan === undefined) {
+      this.userSpeakingSpan = tracer.startSpan({
+        name: 'user_speaking',
+        context: this.rootSpanContext,
+      });
+
+      // TODO(brian): PR4 - Set participant attributes if roomIO.linkedParticipant is available
+      // (Ref: Python agent_session.py line 1192-1195)
+    } else if (this.userSpeakingSpan !== undefined) {
+      // TODO(brian): PR4 - Set ATTR_END_TIME attribute with lastSpeakingTime if available
+      this.userSpeakingSpan.end();
+      this.userSpeakingSpan = undefined;
+    }
+
     const oldState = this.userState;
     this.userState = state;
 
@@ -550,19 +660,33 @@ export class AgentSession<
     reason: CloseReason,
     error: RealtimeModelError | LLMError | TTSError | STTError | null = null,
     drain: boolean = false,
+  ): Promise<void> {
+    if (this.rootSpanContext) {
+      return otelContext.with(this.rootSpanContext, async () => {
+        await this.closeImplInner(reason, error, drain);
+      });
+    }
+
+    return this.closeImplInner(reason, error, drain);
+  }
+
+  private async closeImplInner(
+    reason: CloseReason,
+    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,
+    drain: boolean = false,
   ): Promise<void> {
     if (!this.started) {
       return;
     }
 
     this._cancelUserAwayTimer();
+    this.off(AgentSessionEventTypes.UserInputTranscribed, this._onUserInputTranscribed);
 
     if (this.activity) {
       if (!drain) {
         try {
           this.activity.interrupt();
         } catch (error) {
-          // uninterruptible speech [copied from python]
           // TODO(shubhra): force interrupt or wait for it to finish?
           // it might be an audio played from the error callback
         }
@@ -584,12 +708,28 @@ export class AgentSession<
     await this.activity?.close();
     this.activity = undefined;
 
+    if (this.sessionSpan) {
+      this.sessionSpan.end();
+      this.sessionSpan = undefined;
+    }
+
+    if (this.userSpeakingSpan) {
+      this.userSpeakingSpan.end();
+      this.userSpeakingSpan = undefined;
+    }
+
+    if (this.agentSpeakingSpan) {
+      this.agentSpeakingSpan.end();
+      this.agentSpeakingSpan = undefined;
+    }
+
     this.started = false;
 
     this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));
 
     this.userState = 'listening';
     this._agentState = 'initializing';
+    this.rootSpanContext = undefined;
 
     this.logger.info({ reason, error }, 'AgentSession closed');
   }