@livekit/agents 1.0.44 → 1.0.46

package/src/utils.ts

@@ -126,6 +126,8 @@ export class Future<T = void> {
   #rejectPromise!: (error: Error) => void;
   #done: boolean = false;
   #rejected: boolean = false;
+  #result: T | undefined = undefined;
+  #error: Error | undefined = undefined;
 
   constructor() {
     this.#await = new Promise<T>((resolve, reject) => {
@@ -142,6 +144,18 @@ export class Future<T = void> {
     return this.#done;
   }
 
+  get result(): T {
+    if (!this.#done) {
+      throw new Error('Future is not done');
+    }
+
+    if (this.#rejected) {
+      throw this.#error;
+    }
+
+    return this.#result!;
+  }
+
   /** Whether the future was rejected (cancelled) */
   get rejected() {
     return this.#rejected;
@@ -149,12 +163,14 @@ export class Future<T = void> {
 
   resolve(value: T) {
     this.#done = true;
+    this.#result = value;
     this.#resolvePromise(value);
   }
 
   reject(error: Error) {
     this.#done = true;
     this.#rejected = true;
+    this.#error = error;
     this.#rejectPromise(error);
   }
 }

package/src/vad.ts

@@ -98,14 +98,15 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
   protected closed = false;
   protected inputClosed = false;
 
-  #vad: VAD;
-  #lastActivityTime = BigInt(0);
-  private logger = log();
-  private deferredInputStream: DeferredReadableStream<AudioFrame>;
+  protected vad: VAD;
+  protected lastActivityTime = BigInt(0);
+  protected logger;
+  protected deferredInputStream: DeferredReadableStream<AudioFrame>;
 
   private metricsStream: ReadableStream<VADEvent>;
   constructor(vad: VAD) {
-    this.#vad = vad;
+    this.logger = log();
+    this.vad = vad;
     this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
 
     this.inputWriter = this.input.writable.getWriter();
@@ -155,16 +156,16 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
       switch (value.type) {
         case VADEventType.START_OF_SPEECH:
           inferenceCount++;
-          if (inferenceCount >= 1000 / this.#vad.capabilities.updateInterval) {
-            this.#vad.emit('metrics_collected', {
+          if (inferenceCount >= 1000 / this.vad.capabilities.updateInterval) {
+            this.vad.emit('metrics_collected', {
               type: 'vad_metrics',
               timestamp: Date.now(),
               idleTimeMs: Math.trunc(
-                Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),
+                Number((process.hrtime.bigint() - this.lastActivityTime) / BigInt(1000000)),
               ),
               inferenceDurationTotalMs,
               inferenceCount,
-              label: this.#vad.label,
+              label: this.vad.label,
             });
 
             inferenceCount = 0;
@@ -173,10 +174,10 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
           break;
         case VADEventType.INFERENCE_DONE:
           inferenceDurationTotalMs += Math.round(value.inferenceDuration);
-          this.#lastActivityTime = process.hrtime.bigint();
+          this.lastActivityTime = process.hrtime.bigint();
           break;
         case VADEventType.END_OF_SPEECH:
-          this.#lastActivityTime = process.hrtime.bigint();
+          this.lastActivityTime = process.hrtime.bigint();
           break;
       }
     }

package/src/voice/agent_activity.ts

@@ -74,6 +74,7 @@ import {
 } from './generation.js';
 import type { TimedString } from './io.js';
 import { SpeechHandle } from './speech_handle.js';
+import { setParticipantSpanAttributes } from './utils.js';
 
 const speechHandleStorage = new AsyncLocalStorage<SpeechHandle>();
 
@@ -299,6 +300,9 @@ export class AgentActivity implements RecognitionHooks {
         minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
         maxEndpointingDelay: this.agentSession.options.maxEndpointingDelay,
         rootSpanContext: this.agentSession.rootSpanContext,
+        sttModel: this.stt?.label,
+        sttProvider: this.getSttProvider(),
+        getLinkedParticipant: () => this.agentSession._roomIO?.linkedParticipant,
       });
       this.audioRecognition.start();
       this.started = true;
@@ -335,6 +339,17 @@ export class AgentActivity implements RecognitionHooks {
     return this.agent.stt || this.agentSession.stt;
   }
 
+  private getSttProvider(): string | undefined {
+    const label = this.stt?.label;
+    if (!label) {
+      return undefined;
+    }
+
+    // Heuristic: most labels look like "<provider>-<model>"
+    const [provider] = label.split('-', 1);
+    return provider || label;
+  }
+
   get llm(): LLM | RealtimeModel | undefined {
     return this.agent.llm || this.agentSession.llm;
   }
@@ -1355,6 +1370,11 @@ export class AgentActivity implements RecognitionHooks {
       span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || '');
     }
 
+    const localParticipant = this.agentSession._roomIO?.localParticipant;
+    if (localParticipant) {
+      setParticipantSpanAttributes(span, localParticipant);
+    }
+
     speechHandleStorage.enterWith(speechHandle);
 
     const audioOutput = this.agentSession.output.audioEnabled
@@ -1815,6 +1835,11 @@ export class AgentActivity implements RecognitionHooks {
 
     span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
 
+    const localParticipant = this.agentSession._roomIO?.localParticipant;
+    if (localParticipant) {
+      setParticipantSpanAttributes(span, localParticipant);
+    }
+
     speechHandleStorage.enterWith(speechHandle);
 
     if (!this.realtimeSession) {

package/src/voice/agent_session.ts

@@ -62,6 +62,7 @@ import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io
 import type { UnknownUserData } from './run_context.js';
 import type { SpeechHandle } from './speech_handle.js';
 import { RunResult } from './testing/run_result.js';
+import { setParticipantSpanAttributes } from './utils.js';
 
 export interface VoiceOptions {
   allowInterruptions: boolean;
@@ -131,7 +132,8 @@ export class AgentSession<
   private started = false;
   private userState: UserState = 'listening';
 
-  private roomIO?: RoomIO;
+  /** @internal */
+  _roomIO?: RoomIO;
   private logger = log();
 
   private _chatCtx: ChatContext;
@@ -294,7 +296,7 @@ export class AgentSession<
 
     const tasks: Promise<void>[] = [];
 
-    if (room && !this.roomIO) {
+    if (room && !this._roomIO) {
       // Check for existing input/output configuration and warn if needed
       if (this.input.audio && inputOptions?.audioEnabled !== false) {
         this.logger.warn(
@@ -314,13 +316,13 @@ export class AgentSession<
         );
       }
 
-      this.roomIO = new RoomIO({
+      this._roomIO = new RoomIO({
         agentSession: this,
         room,
         inputOptions,
         outputOptions,
       });
-      this.roomIO.start();
+      this._roomIO.start();
     }
 
     let ctx: JobContext | undefined = undefined;
@@ -700,8 +702,10 @@ export class AgentSession<
           startTime: options?.startTime,
         });
 
-        // TODO(brian): PR4 - Set participant attributes if roomIO.room.localParticipant is available
-        // (Ref: Python agent_session.py line 1161-1164)
+        const localParticipant = this._roomIO?.localParticipant;
+        if (localParticipant) {
+          setParticipantSpanAttributes(this.agentSpeakingSpan, localParticipant);
+        }
       }
     } else if (this.agentSpeakingSpan !== undefined) {
       // TODO(brian): PR4 - Set ATTR_END_TIME attribute if available
@@ -738,8 +742,10 @@ export class AgentSession<
         startTime: lastSpeakingTime,
       });
 
-      // TODO(brian): PR4 - Set participant attributes if roomIO.linkedParticipant is available
-      // (Ref: Python agent_session.py line 1192-1195)
+      const linked = this._roomIO?.linkedParticipant;
+      if (linked) {
+        setParticipantSpanAttributes(this.userSpeakingSpan, linked);
+      }
     } else if (this.userSpeakingSpan !== undefined) {
       this.userSpeakingSpan.end(lastSpeakingTime);
       this.userSpeakingSpan = undefined;
@@ -783,7 +789,7 @@ export class AgentSession<
       return;
     }
 
-    if (this.roomIO && !this.roomIO.isParticipantAvailable) {
+    if (this._roomIO && !this._roomIO.isParticipantAvailable) {
       return;
     }
 
@@ -862,8 +868,8 @@ export class AgentSession<
     this.output.audio = null;
     this.output.transcription = null;
 
-    await this.roomIO?.close();
-    this.roomIO = undefined;
+    await this._roomIO?.close();
+    this._roomIO = undefined;
 
     await this.activity?.close();
     this.activity = undefined;

package/src/voice/audio_recognition.ts

@@ -1,8 +1,15 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { ParticipantKind } from '@livekit/rtc-node';
 import { AudioFrame } from '@livekit/rtc-node';
-import type { Context, Span } from '@opentelemetry/api';
+import {
+  type Context,
+  ROOT_CONTEXT,
+  type Span,
+  context as otelContext,
+  trace,
+} from '@opentelemetry/api';
 import type { WritableStreamDefaultWriter } from 'node:stream/web';
 import { ReadableStream } from 'node:stream/web';
 import { type ChatContext } from '../llm/chat_context.js';
@@ -16,6 +23,7 @@ import { Task, delay } from '../utils.js';
 import { type VAD, type VADEvent, VADEventType } from '../vad.js';
 import type { TurnDetectionMode } from './agent_session.js';
 import type { STTNode } from './io.js';
+import { setParticipantSpanAttributes } from './utils.js';
 
 export interface EndOfTurnInfo {
   /** The new transcript text from the user's speech. */
@@ -72,6 +80,22 @@ export interface AudioRecognitionOptions {
   maxEndpointingDelay: number;
   /** Root span context for tracing. */
   rootSpanContext?: Context;
+  /** STT model name for tracing */
+  sttModel?: string;
+  /** STT provider name for tracing */
+  sttProvider?: string;
+  /** Getter for linked participant for span attribution */
+  getLinkedParticipant?: () => ParticipantLike | undefined;
+}
+
+/**
+ * Minimal participant shape for span attribution.
+ * Compatible with both `LocalParticipant` and `RemoteParticipant` from `@livekit/rtc-node`.
+ */
+export interface ParticipantLike {
+  sid: string | undefined;
+  identity: string;
+  kind: ParticipantKind;
 }
 
 export class AudioRecognition {
@@ -84,6 +108,9 @@ export class AudioRecognition {
   private maxEndpointingDelay: number;
   private lastLanguage?: string;
   private rootSpanContext?: Context;
+  private sttModel?: string;
+  private sttProvider?: string;
+  private getLinkedParticipant?: () => ParticipantLike | undefined;
 
   private deferredInputStream: DeferredReadableStream<AudioFrame>;
   private logger = log();
@@ -121,6 +148,9 @@ export class AudioRecognition {
     this.maxEndpointingDelay = opts.maxEndpointingDelay;
     this.lastLanguage = undefined;
     this.rootSpanContext = opts.rootSpanContext;
+    this.sttModel = opts.sttModel;
+    this.sttProvider = opts.sttProvider;
+    this.getLinkedParticipant = opts.getLinkedParticipant;
 
     this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
     const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
@@ -151,6 +181,37 @@ export class AudioRecognition {
     });
   }
 
+  private ensureUserTurnSpan(startTime?: number): Span {
+    if (this.userTurnSpan && this.userTurnSpan.isRecording()) {
+      return this.userTurnSpan;
+    }
+
+    this.userTurnSpan = tracer.startSpan({
+      name: 'user_turn',
+      context: this.rootSpanContext,
+      startTime,
+    });
+
+    const participant = this.getLinkedParticipant?.();
+    if (participant) {
+      setParticipantSpanAttributes(this.userTurnSpan, participant);
+    }
+
+    if (this.sttModel) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.sttModel);
+    }
+    if (this.sttProvider) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_PROVIDER_NAME, this.sttProvider);
+    }
+
+    return this.userTurnSpan;
+  }
+
+  private userTurnContext(span: Span): Context {
+    const base = this.rootSpanContext ?? ROOT_CONTEXT;
+    return trace.setSpan(base, span);
+  }
+
   private async onSTTEvent(ev: SpeechEvent) {
     if (
       this.turnDetectionMode === 'manual' &&
@@ -299,19 +360,25 @@ export class AudioRecognition {
         break;
       case SpeechEventType.START_OF_SPEECH:
         if (this.turnDetectionMode !== 'stt') break;
-        this.hooks.onStartOfSpeech({
-          type: VADEventType.START_OF_SPEECH,
-          samplesIndex: 0,
-          timestamp: Date.now(),
-          speechDuration: 0,
-          silenceDuration: 0,
-          frames: [],
-          probability: 0,
-          inferenceDuration: 0,
-          speaking: true,
-          rawAccumulatedSilence: 0,
-          rawAccumulatedSpeech: 0,
-        });
+        {
+          const span = this.ensureUserTurnSpan(Date.now());
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onStartOfSpeech({
+              type: VADEventType.START_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: true,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0,
+            });
+          });
+        }
         this.speaking = true;
         this.lastSpeakingTime = Date.now();
 
@@ -319,19 +386,25 @@ export class AudioRecognition {
         break;
       case SpeechEventType.END_OF_SPEECH:
         if (this.turnDetectionMode !== 'stt') break;
-        this.hooks.onEndOfSpeech({
-          type: VADEventType.END_OF_SPEECH,
-          samplesIndex: 0,
-          timestamp: Date.now(),
-          speechDuration: 0,
-          silenceDuration: 0,
-          frames: [],
-          probability: 0,
-          inferenceDuration: 0,
-          speaking: false,
-          rawAccumulatedSilence: 0,
-          rawAccumulatedSpeech: 0,
-        });
+        {
+          const span = this.ensureUserTurnSpan();
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onEndOfSpeech({
+              type: VADEventType.END_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: false,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0,
+            });
+          });
+        }
         this.speaking = false;
         this.userTurnCommitted = true;
         this.lastSpeakingTime = Date.now();
@@ -376,6 +449,9 @@ export class AudioRecognition {
       async (controller: AbortController) => {
         let endpointingDelay = this.minEndpointingDelay;
 
+        const userTurnSpan = this.ensureUserTurnSpan();
+        const userTurnCtx = this.userTurnContext(userTurnSpan);
+
         if (turnDetector) {
           await tracer.startActiveSpan(
             async (span) => {
@@ -415,7 +491,7 @@ export class AudioRecognition {
             },
             {
               name: 'eou_detection',
-              context: this.rootSpanContext,
+              context: userTurnCtx,
             },
           );
         }
@@ -577,17 +653,13 @@ export class AudioRecognition {
         switch (ev.type) {
           case VADEventType.START_OF_SPEECH:
             this.logger.debug('VAD task: START_OF_SPEECH');
-            this.hooks.onStartOfSpeech(ev);
-            this.speaking = true;
-
-            if (!this.userTurnSpan) {
+            {
               const startTime = Date.now() - ev.speechDuration;
-              this.userTurnSpan = tracer.startSpan({
-                name: 'user_turn',
-                context: this.rootSpanContext,
-                startTime,
-              });
+              const span = this.ensureUserTurnSpan(startTime);
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onStartOfSpeech(ev));
             }
+            this.speaking = true;
 
             // Capture sample rate from the first VAD event if not already set
             if (ev.frames.length > 0 && ev.frames[0]) {
@@ -609,7 +681,11 @@ export class AudioRecognition {
             break;
           case VADEventType.END_OF_SPEECH:
             this.logger.debug('VAD task: END_OF_SPEECH');
-            this.hooks.onEndOfSpeech(ev);
+            {
+              const span = this.ensureUserTurnSpan();
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onEndOfSpeech(ev));
+            }
 
             // when VAD fires END_OF_SPEECH, it already waited for the silence_duration
             this.speaking = false;