@livekit/agents 1.0.21 → 1.0.23

package/src/tts/tts.ts

@@ -3,12 +3,14 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { Span } from '@opentelemetry/api';
 import { EventEmitter } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
 import { APIConnectionError, APIError } from '../_exceptions.js';
 import { log } from '../log.js';
 import type { TTSMetrics } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
+import { recordException, traceTypes, tracer } from '../telemetry/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
 import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';
 
@@ -134,6 +136,7 @@ export abstract class SynthesizeStream
   #monitorMetricsTask?: Promise<void>;
   private _connOptions: APIConnectOptions;
   protected abortController = new AbortController();
+  #ttsRequestSpan?: Span;
 
   private deferredInputStream: DeferredReadableStream<
     string | typeof SynthesizeStream.FLUSH_SENTINEL
@@ -160,12 +163,27 @@ export abstract class SynthesizeStream
     startSoon(() => this.mainTask().then(() => this.queue.close()));
   }
 
-  private async mainTask() {
-    // TODO(brian): PR3 - Add span wrapping: tracer.startActiveSpan('tts_request', ..., { endOnExit: false })
+  private _mainTaskImpl = async (span: Span) => {
+    this.#ttsRequestSpan = span;
+    span.setAttributes({
+      [traceTypes.ATTR_TTS_STREAMING]: true,
+      [traceTypes.ATTR_TTS_LABEL]: this.#tts.label,
+    });
+
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
-        // TODO(brian): PR3 - Add span for retry attempts: tracer.startActiveSpan('tts_request_run', ...)
-        return await this.run();
+        return await tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              recordException(attemptSpan, toError(error));
+              throw error;
+            }
+          },
+          { name: 'tts_request_run' },
+        );
       } catch (error) {
         if (error instanceof APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
@@ -197,7 +215,13 @@ export abstract class SynthesizeStream
         }
       }
     }
-  }
+  };
+
+  private mainTask = async () =>
+    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+      name: 'tts_request',
+      endOnExit: false,
+    });
 
   private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
     this.#tts.emit('error', {
@@ -265,6 +289,9 @@ export abstract class SynthesizeStream
           label: this.#tts.label,
           streamed: false,
         };
+        if (this.#ttsRequestSpan) {
+          this.#ttsRequestSpan.setAttribute(traceTypes.ATTR_TTS_METRICS, JSON.stringify(metrics));
+        }
         this.#tts.emit('metrics_collected', metrics);
       }
     };
@@ -289,6 +316,11 @@ export abstract class SynthesizeStream
     if (requestId) {
       emit();
     }
+
+    if (this.#ttsRequestSpan) {
+      this.#ttsRequestSpan.end();
+      this.#ttsRequestSpan = undefined;
+    }
   }
 
   protected abstract run(): Promise<void>;
@@ -307,12 +339,11 @@ export abstract class SynthesizeStream
     }
     this.#metricsText += text;
 
-    if (this.input.closed) {
-      throw new Error('Input is closed');
-    }
-    if (this.closed) {
-      throw new Error('Stream is closed');
+    if (this.input.closed || this.closed) {
+      // Stream was aborted/closed, silently skip
+      return;
     }
+
     this.input.put(text);
   }
 
@@ -322,24 +353,24 @@ export abstract class SynthesizeStream
       this.#metricsPendingTexts.push(this.#metricsText);
       this.#metricsText = '';
     }
-    if (this.input.closed) {
-      throw new Error('Input is closed');
-    }
-    if (this.closed) {
-      throw new Error('Stream is closed');
+
+    if (this.input.closed || this.closed) {
+      // Stream was aborted/closed, silently skip
+      return;
     }
+
     this.input.put(SynthesizeStream.FLUSH_SENTINEL);
   }
 
   /** Mark the input as ended and forbid additional pushes */
   endInput() {
     this.flush();
-    if (this.input.closed) {
-      throw new Error('Input is closed');
-    }
-    if (this.closed) {
-      throw new Error('Stream is closed');
+
+    if (this.input.closed || this.closed) {
+      // Stream was aborted/closed, silently skip
+      return;
     }
+
     this.input.close();
   }
 
@@ -378,6 +409,7 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
   abstract label: string;
   #text: string;
   #tts: TTS;
+  #ttsRequestSpan?: Span;
   private _connOptions: APIConnectOptions;
   private logger = log();
 
@@ -399,12 +431,27 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
     Promise.resolve().then(() => this.mainTask().then(() => this.queue.close()));
   }
 
-  private async mainTask() {
-    // TODO(brian): PR3 - Add span wrapping: tracer.startActiveSpan('tts_request', ..., { endOnExit: false })
+  private _mainTaskImpl = async (span: Span) => {
+    this.#ttsRequestSpan = span;
+    span.setAttributes({
+      [traceTypes.ATTR_TTS_STREAMING]: false,
+      [traceTypes.ATTR_TTS_LABEL]: this.#tts.label,
+    });
+
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
-        // TODO(brian): PR3 - Add span for retry attempts: tracer.startActiveSpan('tts_request_run', ...)
-        return await this.run();
+        return await tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              recordException(attemptSpan, toError(error));
+              throw error;
+            }
+          },
+          { name: 'tts_request_run' },
+        );
       } catch (error) {
         if (error instanceof APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
@@ -436,6 +483,13 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
         }
       }
     }
+  };
+
+  private async mainTask() {
+    return tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+      name: 'tts_request',
+      endOnExit: false,
+    });
   }
 
   private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
@@ -483,6 +537,13 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
       label: this.#tts.label,
       streamed: false,
     };
+
+    if (this.#ttsRequestSpan) {
+      this.#ttsRequestSpan.setAttribute(traceTypes.ATTR_TTS_METRICS, JSON.stringify(metrics));
+      this.#ttsRequestSpan.end();
+      this.#ttsRequestSpan = undefined;
+    }
+
     this.#tts.emit('metrics_collected', metrics);
   }
 

package/src/utils.ts

@@ -839,3 +839,8 @@ export async function waitForAbort(signal: AbortSignal) {
   signal.addEventListener('abort', handler, { once: true });
   return await abortFuture.await;
 }
+
+export const isCloud = (url: URL) => {
+  const hostname = url.hostname;
+  return hostname.endsWith('.livekit.cloud') || hostname.endsWith('.livekit.run');
+};

package/src/voice/agent_activity.ts

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 import { Mutex } from '@livekit/mutex';
 import type { AudioFrame } from '@livekit/rtc-node';
+import type { Span } from '@opentelemetry/api';
+import { ROOT_CONTEXT, trace } from '@opentelemetry/api';
 import { Heap } from 'heap-js';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { ReadableStream } from 'node:stream/web';
@@ -10,6 +12,7 @@ import { type ChatContext, ChatMessage } from '../llm/chat_context.js';
 import {
   type ChatItem,
   type FunctionCall,
+  type FunctionCallOutput,
   type GenerationCreatedEvent,
   type InputSpeechStartedEvent,
   type InputSpeechStoppedEvent,
@@ -34,6 +37,7 @@ import type {
 } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { STT, type STTError, type SpeechEvent } from '../stt/stt.js';
+import { traceTypes, tracer } from '../telemetry/index.js';
 import { splitWords } from '../tokenize/basic/word.js';
 import { TTS, type TTSError } from '../tts/tts.js';
 import { Future, Task, cancelAndWait, waitFor } from '../utils.js';
@@ -70,7 +74,6 @@ import {
 } from './generation.js';
 import { SpeechHandle } from './speech_handle.js';
 
-// equivalent to Python's contextvars
 const speechHandleStorage = new AsyncLocalStorage<SpeechHandle>();
 
 interface PreemptiveGeneration {
@@ -202,10 +205,15 @@ export class AgentActivity implements RecognitionHooks {
   }
 
   async start(): Promise<void> {
-    // TODO(brian): PR3 - Add span: startSpan = tracer.startSpan('start_agent_activity', { attributes: { 'lk.agent_label': this.agent.label } })
-    // TODO(brian): PR3 - Wrap prewarm calls with trace.useSpan(startSpan, endOnExit: false)
     const unlock = await this.lock.lock();
     try {
+      // Create start_agent_activity as a ROOT span (new trace) to match Python behavior
+      const startSpan = tracer.startSpan({
+        name: 'start_agent_activity',
+        attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+        context: ROOT_CONTEXT,
+      });
+
       this.agent._agentActivity = this;
 
       if (this.llm instanceof RealtimeModel) {
@@ -286,16 +294,26 @@ export class AgentActivity implements RecognitionHooks {
         turnDetectionMode: this.turnDetectionMode,
         minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
         maxEndpointingDelay: this.agentSession.options.maxEndpointingDelay,
+        rootSpanContext: this.agentSession.rootSpanContext,
       });
       this.audioRecognition.start();
       this.started = true;
 
       this._mainTask = Task.from(({ signal }) => this.mainTask(signal));
-      // TODO(brian): PR3 - Wrap onEnter with tracer.startActiveSpan('on_enter', { attributes: { 'lk.agent_label': this.agent.label }, context: startSpan context })
+
+      // Create on_enter as a child of start_agent_activity in the new trace
+      const onEnterTask = tracer.startActiveSpan(async () => this.agent.onEnter(), {
+        name: 'on_enter',
+        context: trace.setSpan(ROOT_CONTEXT, startSpan),
+        attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+      });
+
       this.createSpeechTask({
-        task: Task.from(() => this.agent.onEnter()),
+        task: Task.from(() => onEnterTask),
         name: 'AgentActivity_onEnter',
       });
+
+      startSpan.end();
     } finally {
       unlock();
     }
@@ -577,7 +595,6 @@ export class AgentActivity implements RecognitionHooks {
     }
 
     if (this.draining) {
-      // copied from python:
       // TODO(shubhra): should we "forward" this new turn to the next agent?
       this.logger.warn('skipping new realtime generation, the agent is draining');
       return;
@@ -783,7 +800,6 @@ export class AgentActivity implements RecognitionHooks {
     if (this.draining) {
       this.cancelPreemptiveGeneration();
       this.logger.warn({ user_input: info.newTranscript }, 'skipping user input, task is draining');
-      // copied from python:
       // TODO(shubhra): should we "forward" this new turn to the next agent/activity?
       return true;
     }
@@ -1254,17 +1270,35 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
-  // TODO(brian): PR3 - Wrap entire pipelineReplyTask() method with tracer.startActiveSpan('agent_turn')
-  private async pipelineReplyTask(
-    speechHandle: SpeechHandle,
-    chatCtx: ChatContext,
-    toolCtx: ToolContext,
-    modelSettings: ModelSettings,
-    replyAbortController: AbortController,
-    instructions?: string,
-    newMessage?: ChatMessage,
-    toolsMessages?: ChatItem[],
-  ): Promise<void> {
+  private _pipelineReplyTaskImpl = async ({
+    speechHandle,
+    chatCtx,
+    toolCtx,
+    modelSettings,
+    replyAbortController,
+    instructions,
+    newMessage,
+    toolsMessages,
+    span,
+  }: {
+    speechHandle: SpeechHandle;
+    chatCtx: ChatContext;
+    toolCtx: ToolContext;
+    modelSettings: ModelSettings;
+    replyAbortController: AbortController;
+    instructions?: string;
+    newMessage?: ChatMessage;
+    toolsMessages?: ChatItem[];
+    span: Span;
+  }): Promise<void> => {
+    span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
+    if (instructions) {
+      span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, instructions);
+    }
+    if (newMessage) {
+      span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || '');
+    }
+
     speechHandleStorage.enterWith(speechHandle);
 
     const audioOutput = this.agentSession.output.audioEnabled
@@ -1406,6 +1440,8 @@ export class AgentActivity implements RecognitionHooks {
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolsMessages);
+      // Also add to session history (matches Python agent_session.py _tool_items_added)
+      this.agentSession._toolItemsAdded(toolsMessages as (FunctionCall | FunctionCallOutput)[]);
     }
 
     if (speechHandle.interrupted) {
@@ -1601,8 +1637,38 @@ export class AgentActivity implements RecognitionHooks {
         msg.createdAt = replyStartedAt;
       }
       this.agent._chatCtx.insert(toolMessages);
+      this.agentSession._toolItemsAdded(toolMessages as (FunctionCall | FunctionCallOutput)[]);
     }
-  }
+  };
+
+  private pipelineReplyTask = async (
+    speechHandle: SpeechHandle,
+    chatCtx: ChatContext,
+    toolCtx: ToolContext,
+    modelSettings: ModelSettings,
+    replyAbortController: AbortController,
+    instructions?: string,
+    newMessage?: ChatMessage,
+    toolsMessages?: ChatItem[],
+  ): Promise<void> =>
+    tracer.startActiveSpan(
+      async (span) =>
+        this._pipelineReplyTaskImpl({
+          speechHandle,
+          chatCtx,
+          toolCtx,
+          modelSettings,
+          replyAbortController,
+          instructions,
+          newMessage,
+          toolsMessages,
+          span,
+        }),
+      {
+        name: 'agent_turn',
+        context: this.agentSession.rootSpanContext,
+      },
+    );
 
   private async realtimeGenerationTask(
     speechHandle: SpeechHandle,
@@ -1610,6 +1676,37 @@ export class AgentActivity implements RecognitionHooks {
     modelSettings: ModelSettings,
     replyAbortController: AbortController,
   ): Promise<void> {
+    return tracer.startActiveSpan(
+      async (span) =>
+        this._realtimeGenerationTaskImpl({
+          speechHandle,
+          ev,
+          modelSettings,
+          replyAbortController,
+          span,
+        }),
+      {
+        name: 'agent_turn',
+        context: this.agentSession.rootSpanContext,
+      },
+    );
+  }
+
+  private async _realtimeGenerationTaskImpl({
+    speechHandle,
+    ev,
+    modelSettings,
+    replyAbortController,
+    span,
+  }: {
+    speechHandle: SpeechHandle;
+    ev: GenerationCreatedEvent;
+    modelSettings: ModelSettings;
+    replyAbortController: AbortController;
+    span: Span;
+  }): Promise<void> {
+    span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
+
     speechHandleStorage.enterWith(speechHandle);
 
     if (!this.realtimeSession) {
@@ -1786,6 +1883,8 @@ export class AgentActivity implements RecognitionHooks {
 
     const onToolExecutionStarted = (f: FunctionCall) => {
       speechHandle._itemAdded([f]);
+      this.agent._chatCtx.items.push(f);
+      this.agentSession._toolItemsAdded([f]);
     };
 
     const onToolExecutionCompleted = (out: ToolExecutionOutput) => {
@@ -1979,6 +2078,11 @@ export class AgentActivity implements RecognitionHooks {
       }
       const chatCtx = this.realtimeSession.chatCtx.copy();
       chatCtx.items.push(...functionToolsExecutedEvent.functionCallOutputs);
+
+      this.agentSession._toolItemsAdded(
+        functionToolsExecutedEvent.functionCallOutputs as FunctionCallOutput[],
+      );
+
       try {
         await this.realtimeSession.updateChatCtx(chatCtx);
       } catch (error) {
@@ -2096,16 +2200,30 @@ export class AgentActivity implements RecognitionHooks {
     this.wakeupMainTask();
   }
 
-  // TODO(brian): PR3 - Wrap entire drain() method with tracer.startActiveSpan('drain_agent_activity', { attributes: { 'lk.agent_label': this.agent.label } })
   async drain(): Promise<void> {
+    // Create drain_agent_activity as a ROOT span (new trace) to match Python behavior
+    return tracer.startActiveSpan(async (span) => this._drainImpl(span), {
+      name: 'drain_agent_activity',
+      context: ROOT_CONTEXT,
+    });
+  }
+
+  private async _drainImpl(span: Span): Promise<void> {
+    span.setAttribute(traceTypes.ATTR_AGENT_LABEL, this.agent.id);
+
     const unlock = await this.lock.lock();
     try {
       if (this._draining) return;
 
       this.cancelPreemptiveGeneration();
-      // TODO(brian): PR3 - Wrap onExit with tracer.startActiveSpan('on_exit', { attributes: { 'lk.agent_label': this.agent.label } })
+
+      const onExitTask = tracer.startActiveSpan(async () => this.agent.onExit(), {
+        name: 'on_exit',
+        attributes: { [traceTypes.ATTR_AGENT_LABEL]: this.agent.id },
+      });
+
       this.createSpeechTask({
-        task: Task.from(() => this.agent.onExit()),
+        task: Task.from(() => onExitTask),
         name: 'AgentActivity_onExit',
       });