@livekit/agents 1.0.45 → 1.0.47

package/src/voice/agent_session.ts

@@ -1,12 +1,14 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import { Mutex } from '@livekit/mutex';
 import type { AudioFrame, Room } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { Context, Span } from '@opentelemetry/api';
 import { ROOT_CONTEXT, context as otelContext, trace } from '@opentelemetry/api';
 import { EventEmitter } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
+import type { z } from 'zod';
 import {
   LLM as InferenceLLM,
   STT as InferenceSTT,
@@ -31,6 +33,7 @@ import {
   type ResolvedSessionConnectOptions,
   type SessionConnectOptions,
 } from '../types.js';
+import { Task } from '../utils.js';
 import type { VAD } from '../vad.js';
 import type { Agent } from './agent.js';
 import { AgentActivity } from './agent_activity.js';
@@ -62,6 +65,7 @@ import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io
 import type { UnknownUserData } from './run_context.js';
 import type { SpeechHandle } from './speech_handle.js';
 import { RunResult } from './testing/run_result.js';
+import { setParticipantSpanAttributes } from './utils.js';
 
 export interface VoiceOptions {
   allowInterruptions: boolean;
@@ -114,6 +118,13 @@ export type AgentSessionOptions<UserData = UnknownUserData> = {
   connOptions?: SessionConnectOptions;
 };
 
+type ActivityTransitionOptions = {
+  previousActivity?: 'close' | 'pause';
+  newActivity?: 'start' | 'resume';
+  blockedTasks?: Task<any>[];
+  waitOnEnter?: boolean;
+};
+
 export class AgentSession<
   UserData = UnknownUserData,
 > extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {
@@ -128,10 +139,13 @@ export class AgentSession<
   private agent?: Agent;
   private activity?: AgentActivity;
   private nextActivity?: AgentActivity;
+  private updateActivityTask?: Task<void>;
   private started = false;
   private userState: UserState = 'listening';
+  private readonly activityLock = new Mutex();
 
-  private roomIO?: RoomIO;
+  /** @internal */
+  _roomIO?: RoomIO;
   private logger = log();
 
   private _chatCtx: ChatContext;
@@ -294,7 +308,7 @@ export class AgentSession<
 
     const tasks: Promise<void>[] = [];
 
-    if (room && !this.roomIO) {
+    if (room && !this._roomIO) {
       // Check for existing input/output configuration and warn if needed
       if (this.input.audio && inputOptions?.audioEnabled !== false) {
         this.logger.warn(
@@ -314,13 +328,13 @@ export class AgentSession<
         );
       }
 
-      this.roomIO = new RoomIO({
+      this._roomIO = new RoomIO({
         agentSession: this,
         room,
         inputOptions,
         outputOptions,
       });
-      this.roomIO.start();
+      this._roomIO.start();
     }
 
     let ctx: JobContext | undefined = undefined;
@@ -358,7 +372,8 @@ export class AgentSession<
     }
 
     // TODO(AJS-265): add shutdown callback to job context
-    tasks.push(this.updateActivity(this.agent));
+    // Initial start does not wait on onEnter
+    tasks.push(this._updateActivity(this.agent, { waitOnEnter: false }));
 
     await Promise.allSettled(tasks);
 
@@ -430,8 +445,34 @@ export class AgentSession<
   updateAgent(agent: Agent): void {
     this.agent = agent;
 
-    if (this.started) {
-      this.updateActivity(agent);
+    if (!this.started) {
+      return;
+    }
+
+    const _updateActivityTask = async (oldTask: Task<void> | undefined, agent: Agent) => {
+      if (oldTask) {
+        try {
+          await oldTask.result;
+        } catch (error) {
+          this.logger.error(error, 'previous updateAgent transition failed');
+        }
+      }
+
+      await this._updateActivity(agent);
+    };
+
+    const oldTask = this.updateActivityTask;
+    this.updateActivityTask = Task.from(
+      async () => _updateActivityTask(oldTask, agent),
+      undefined,
+      'AgentSession_updateActivityTask',
+    );
+
+    const runState = this._globalRunState;
+    if (runState) {
+      // Don't mark the RunResult as done, if there is currently an agent transition happening.
+      // (used to make sure we're correctly adding the AgentHandoffResult before completion)
+      runState._watchHandle(this.updateActivityTask);
     }
   }
 
@@ -462,24 +503,42 @@ export class AgentSession<
       throw new Error('AgentSession is not running');
     }
 
-    const doSay = (activity: AgentActivity) => {
+    const doSay = (activity: AgentActivity, nextActivity?: AgentActivity) => {
+      if (activity.schedulingPaused) {
+        if (!nextActivity) {
+          throw new Error('AgentSession is closing, cannot use say()');
+        }
+        return nextActivity.say(text, options);
+      }
       return activity.say(text, options);
     };
 
+    const runState = this._globalRunState;
+    let handle: SpeechHandle;
+
     // attach to the session span if called outside of the AgentSession
     const activeSpan = trace.getActiveSpan();
     if (!activeSpan && this.rootSpanContext) {
-      return otelContext.with(this.rootSpanContext, () => doSay(this.activity!));
+      handle = otelContext.with(this.rootSpanContext, () =>
+        doSay(this.activity!, this.nextActivity),
+      );
+    } else {
+      handle = doSay(this.activity, this.nextActivity);
+    }
+
+    if (runState) {
+      runState._watchHandle(handle);
     }
 
-    return doSay(this.activity);
+    return handle;
   }
 
-  interrupt() {
+  interrupt(options?: { force?: boolean }) {
     if (!this.activity) {
       throw new Error('AgentSession is not running');
     }
-    return this.activity.interrupt();
+
+    return this.activity.interrupt(options);
   }
 
   generateReply(options?: {
@@ -500,7 +559,7 @@ export class AgentSession<
       : undefined;
 
     const doGenerateReply = (activity: AgentActivity, nextActivity?: AgentActivity) => {
-      if (activity.draining) {
+      if (activity.schedulingPaused) {
         if (!nextActivity) {
           throw new Error('AgentSession is closing, cannot use generateReply()');
         }
@@ -540,53 +599,128 @@ export class AgentSession<
    * result.expect.noMoreEvents();
    * ```
    *
-   * @param options - Run options including user input
+   * @param options - Run options including user input and optional output type
    * @returns A RunResult that resolves when the agent finishes responding
-   *
-   * TODO: Add outputType parameter for typed outputs (parity with Python)
    */
-  run(options: { userInput: string }): RunResult {
+  run<T = unknown>({
+    userInput,
+    outputType,
+  }: {
+    userInput: string;
+    outputType?: z.ZodType<T>;
+  }): RunResult<T> {
     if (this._globalRunState && !this._globalRunState.done()) {
       throw new Error('nested runs are not supported');
     }
 
-    const runState = new RunResult({ userInput: options.userInput });
+    const runState = new RunResult<T>({
+      userInput,
+      outputType,
+    });
+
     this._globalRunState = runState;
-    this.generateReply({ userInput: options.userInput });
+
+    // Defer generateReply through the activityLock to ensure any in-progress
+    // activity transition (e.g. AgentTask started from onEnter) completes first.
+    // TS Task.from starts onEnter synchronously, so the transition may already be
+    // mid-flight by the time run() is called after session.start() resolves.
+    // Acquiring and immediately releasing the lock guarantees FIFO ordering:
+    // the transition's lock section finishes before we route generateReply.
+    (async () => {
+      try {
+        const unlock = await this.activityLock.lock();
+        unlock();
+        this.generateReply({ userInput });
+      } catch (e) {
+        runState._reject(e instanceof Error ? e : new Error(String(e)));
+      }
+    })();
 
     return runState;
   }
 
-  private async updateActivity(agent: Agent): Promise<void> {
+  /** @internal */
+  async _updateActivity(agent: Agent, options: ActivityTransitionOptions = {}): Promise<void> {
+    const { previousActivity = 'close', newActivity = 'start', blockedTasks = [] } = options;
+    const waitOnEnter = options.waitOnEnter ?? newActivity === 'start';
+
     const runWithContext = async () => {
-      // TODO(AJS-129): add lock to agent activity core lifecycle
-      this.nextActivity = new AgentActivity(agent, this);
+      const unlock = await this.activityLock.lock();
+      let onEnterTask: Task<void> | undefined;
 
-      const previousActivity = this.activity;
+      try {
+        this.agent = agent;
+        const prevActivityObj = this.activity;
+
+        if (newActivity === 'start') {
+          const prevAgent = prevActivityObj?.agent;
+          if (
+            agent._agentActivity &&
+            // allow updating the same agent that is running
+            (agent !== prevAgent || previousActivity !== 'close')
+          ) {
+            throw new Error('Cannot start agent: an activity is already running');
+          }
+          this.nextActivity = new AgentActivity(agent, this);
+        } else if (newActivity === 'resume') {
+          if (!agent._agentActivity) {
+            throw new Error('Cannot resume agent: no existing activity to resume');
+          }
+          this.nextActivity = agent._agentActivity;
+        }
 
-      if (this.activity) {
-        await this.activity.drain();
-        await this.activity.close();
-      }
+        if (prevActivityObj && prevActivityObj !== this.nextActivity) {
+          if (previousActivity === 'pause') {
+            await prevActivityObj.pause({ blockedTasks });
+          } else {
+            await prevActivityObj.drain();
+            await prevActivityObj.close();
+          }
+        }
 
-      this.activity = this.nextActivity;
-      this.nextActivity = undefined;
+        this.activity = this.nextActivity;
+        this.nextActivity = undefined;
 
-      this._chatCtx.insert(
-        new AgentHandoffItem({
-          oldAgentId: previousActivity?.agent.id,
+        const runState = this._globalRunState;
+        const handoffItem = new AgentHandoffItem({
+          oldAgentId: prevActivityObj?.agent.id,
           newAgentId: agent.id,
-        }),
-      );
-      this.logger.debug(
-        { previousAgentId: previousActivity?.agent.id, newAgentId: agent.id },
-        'Agent handoff inserted into chat context',
-      );
+        });
+
+        if (runState) {
+          runState._agentHandoff({
+            item: handoffItem,
+            oldAgent: prevActivityObj?.agent,
+            newAgent: this.activity!.agent,
+          });
+        }
+
+        this._chatCtx.insert(handoffItem);
+        this.logger.debug(
+          { previousAgentId: prevActivityObj?.agent.id, newAgentId: agent.id },
+          'Agent handoff inserted into chat context',
+        );
+
+        if (newActivity === 'start') {
+          await this.activity!.start();
+        } else {
+          await this.activity!.resume();
+        }
 
-      await this.activity.start();
+        onEnterTask = this.activity!._onEnterTask;
 
-      if (this._input.audio) {
-        this.activity.attachAudioInput(this._input.audio.stream);
+        if (this._input.audio) {
+          this.activity!.attachAudioInput(this._input.audio.stream);
+        }
+      } finally {
+        unlock();
+      }
+
+      if (waitOnEnter) {
+        if (!onEnterTask) {
+          throw new Error('expected onEnter task to be available while waitOnEnter=true');
+        }
+        await onEnterTask.result;
       }
     };
 
@@ -700,8 +834,10 @@ export class AgentSession<
           startTime: options?.startTime,
         });
 
-        // TODO(brian): PR4 - Set participant attributes if roomIO.room.localParticipant is available
-        // (Ref: Python agent_session.py line 1161-1164)
+        const localParticipant = this._roomIO?.localParticipant;
+        if (localParticipant) {
+          setParticipantSpanAttributes(this.agentSpeakingSpan, localParticipant);
+        }
       }
     } else if (this.agentSpeakingSpan !== undefined) {
       // TODO(brian): PR4 - Set ATTR_END_TIME attribute if available
@@ -738,8 +874,10 @@ export class AgentSession<
         startTime: lastSpeakingTime,
       });
 
-      // TODO(brian): PR4 - Set participant attributes if roomIO.linkedParticipant is available
-      // (Ref: Python agent_session.py line 1192-1195)
+      const linked = this._roomIO?.linkedParticipant;
+      if (linked) {
+        setParticipantSpanAttributes(this.userSpeakingSpan, linked);
+      }
     } else if (this.userSpeakingSpan !== undefined) {
       this.userSpeakingSpan.end(lastSpeakingTime);
       this.userSpeakingSpan = undefined;
@@ -783,7 +921,7 @@ export class AgentSession<
       return;
     }
 
-    if (this.roomIO && !this.roomIO.isParticipantAvailable) {
+    if (this._roomIO && !this._roomIO.isParticipantAvailable) {
       return;
     }
 
@@ -836,15 +974,21 @@ export class AgentSession<
     if (this.activity) {
       if (!drain) {
         try {
-          this.activity.interrupt();
+          await this.activity.interrupt({ force: true }).await;
         } catch (error) {
-          // TODO(shubhra): force interrupt or wait for it to finish?
-          // it might be an audio played from the error callback
+          // Uninterruptible speech can throw during forced interruption.
+          this.logger.warn({ error }, 'Error interrupting activity');
         }
       }
+
       await this.activity.drain();
       // wait any uninterruptible speech to finish
       await this.activity.currentSpeech?.waitForPlayout();
+
+      if (reason !== CloseReason.ERROR) {
+        this.activity.commitUserTurn({ audioDetached: true, throwIfNotReady: false });
+      }
+
       try {
         this.activity.detachAudioInput();
       } catch (error) {
@@ -862,8 +1006,8 @@ export class AgentSession<
     this.output.audio = null;
     this.output.transcription = null;
 
-    await this.roomIO?.close();
-    this.roomIO = undefined;
+    await this._roomIO?.close();
+    this._roomIO = undefined;
 
     await this.activity?.close();
     this.activity = undefined;

package/src/voice/audio_recognition.ts

@@ -1,8 +1,15 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { ParticipantKind } from '@livekit/rtc-node';
 import { AudioFrame } from '@livekit/rtc-node';
-import type { Context, Span } from '@opentelemetry/api';
+import {
+  type Context,
+  ROOT_CONTEXT,
+  type Span,
+  context as otelContext,
+  trace,
+} from '@opentelemetry/api';
 import type { WritableStreamDefaultWriter } from 'node:stream/web';
 import { ReadableStream } from 'node:stream/web';
 import { type ChatContext } from '../llm/chat_context.js';
@@ -16,6 +23,7 @@ import { Task, delay } from '../utils.js';
 import { type VAD, type VADEvent, VADEventType } from '../vad.js';
 import type { TurnDetectionMode } from './agent_session.js';
 import type { STTNode } from './io.js';
+import { setParticipantSpanAttributes } from './utils.js';
 
 export interface EndOfTurnInfo {
   /** The new transcript text from the user's speech. */
@@ -72,6 +80,22 @@ export interface AudioRecognitionOptions {
   maxEndpointingDelay: number;
   /** Root span context for tracing. */
   rootSpanContext?: Context;
+  /** STT model name for tracing */
+  sttModel?: string;
+  /** STT provider name for tracing */
+  sttProvider?: string;
+  /** Getter for linked participant for span attribution */
+  getLinkedParticipant?: () => ParticipantLike | undefined;
+}
+
+/**
+ * Minimal participant shape for span attribution.
+ * Compatible with both `LocalParticipant` and `RemoteParticipant` from `@livekit/rtc-node`.
+ */
+export interface ParticipantLike {
+  sid: string | undefined;
+  identity: string;
+  kind: ParticipantKind;
 }
 
 export class AudioRecognition {
@@ -84,6 +108,9 @@ export class AudioRecognition {
   private maxEndpointingDelay: number;
   private lastLanguage?: string;
   private rootSpanContext?: Context;
+  private sttModel?: string;
+  private sttProvider?: string;
+  private getLinkedParticipant?: () => ParticipantLike | undefined;
 
   private deferredInputStream: DeferredReadableStream<AudioFrame>;
   private logger = log();
@@ -121,6 +148,9 @@ export class AudioRecognition {
     this.maxEndpointingDelay = opts.maxEndpointingDelay;
     this.lastLanguage = undefined;
     this.rootSpanContext = opts.rootSpanContext;
+    this.sttModel = opts.sttModel;
+    this.sttProvider = opts.sttProvider;
+    this.getLinkedParticipant = opts.getLinkedParticipant;
 
     this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
     const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
@@ -151,6 +181,37 @@ export class AudioRecognition {
     });
   }
 
+  private ensureUserTurnSpan(startTime?: number): Span {
+    if (this.userTurnSpan && this.userTurnSpan.isRecording()) {
+      return this.userTurnSpan;
+    }
+
+    this.userTurnSpan = tracer.startSpan({
+      name: 'user_turn',
+      context: this.rootSpanContext,
+      startTime,
+    });
+
+    const participant = this.getLinkedParticipant?.();
+    if (participant) {
+      setParticipantSpanAttributes(this.userTurnSpan, participant);
+    }
+
+    if (this.sttModel) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.sttModel);
+    }
+    if (this.sttProvider) {
+      this.userTurnSpan.setAttribute(traceTypes.ATTR_GEN_AI_PROVIDER_NAME, this.sttProvider);
+    }
+
+    return this.userTurnSpan;
+  }
+
+  private userTurnContext(span: Span): Context {
+    const base = this.rootSpanContext ?? ROOT_CONTEXT;
+    return trace.setSpan(base, span);
+  }
+
   private async onSTTEvent(ev: SpeechEvent) {
     if (
       this.turnDetectionMode === 'manual' &&
@@ -299,19 +360,25 @@ export class AudioRecognition {
         break;
       case SpeechEventType.START_OF_SPEECH:
         if (this.turnDetectionMode !== 'stt') break;
-        this.hooks.onStartOfSpeech({
-          type: VADEventType.START_OF_SPEECH,
-          samplesIndex: 0,
-          timestamp: Date.now(),
-          speechDuration: 0,
-          silenceDuration: 0,
-          frames: [],
-          probability: 0,
-          inferenceDuration: 0,
-          speaking: true,
-          rawAccumulatedSilence: 0,
-          rawAccumulatedSpeech: 0,
-        });
+        {
+          const span = this.ensureUserTurnSpan(Date.now());
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onStartOfSpeech({
+              type: VADEventType.START_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: true,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0,
+            });
+          });
+        }
         this.speaking = true;
         this.lastSpeakingTime = Date.now();
 
@@ -319,19 +386,25 @@ export class AudioRecognition {
         break;
       case SpeechEventType.END_OF_SPEECH:
         if (this.turnDetectionMode !== 'stt') break;
-        this.hooks.onEndOfSpeech({
-          type: VADEventType.END_OF_SPEECH,
-          samplesIndex: 0,
-          timestamp: Date.now(),
-          speechDuration: 0,
-          silenceDuration: 0,
-          frames: [],
-          probability: 0,
-          inferenceDuration: 0,
-          speaking: false,
-          rawAccumulatedSilence: 0,
-          rawAccumulatedSpeech: 0,
-        });
+        {
+          const span = this.ensureUserTurnSpan();
+          const ctx = this.userTurnContext(span);
+          otelContext.with(ctx, () => {
+            this.hooks.onEndOfSpeech({
+              type: VADEventType.END_OF_SPEECH,
+              samplesIndex: 0,
+              timestamp: Date.now(),
+              speechDuration: 0,
+              silenceDuration: 0,
+              frames: [],
+              probability: 0,
+              inferenceDuration: 0,
+              speaking: false,
+              rawAccumulatedSilence: 0,
+              rawAccumulatedSpeech: 0,
+            });
+          });
+        }
         this.speaking = false;
         this.userTurnCommitted = true;
         this.lastSpeakingTime = Date.now();
@@ -376,6 +449,9 @@ export class AudioRecognition {
       async (controller: AbortController) => {
         let endpointingDelay = this.minEndpointingDelay;
 
+        const userTurnSpan = this.ensureUserTurnSpan();
+        const userTurnCtx = this.userTurnContext(userTurnSpan);
+
         if (turnDetector) {
           await tracer.startActiveSpan(
             async (span) => {
@@ -415,7 +491,7 @@ export class AudioRecognition {
             },
             {
               name: 'eou_detection',
-              context: this.rootSpanContext,
+              context: userTurnCtx,
             },
           );
         }
@@ -577,17 +653,13 @@ export class AudioRecognition {
         switch (ev.type) {
           case VADEventType.START_OF_SPEECH:
             this.logger.debug('VAD task: START_OF_SPEECH');
-            this.hooks.onStartOfSpeech(ev);
-            this.speaking = true;
-
-            if (!this.userTurnSpan) {
+            {
               const startTime = Date.now() - ev.speechDuration;
-              this.userTurnSpan = tracer.startSpan({
-                name: 'user_turn',
-                context: this.rootSpanContext,
-                startTime,
-              });
+              const span = this.ensureUserTurnSpan(startTime);
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onStartOfSpeech(ev));
             }
+            this.speaking = true;
 
             // Capture sample rate from the first VAD event if not already set
             if (ev.frames.length > 0 && ev.frames[0]) {
@@ -609,7 +681,11 @@ export class AudioRecognition {
             break;
           case VADEventType.END_OF_SPEECH:
             this.logger.debug('VAD task: END_OF_SPEECH');
-            this.hooks.onEndOfSpeech(ev);
+            {
+              const span = this.ensureUserTurnSpan();
+              const ctx = this.userTurnContext(span);
+              otelContext.with(ctx, () => this.hooks.onEndOfSpeech(ev));
+            }
 
             // when VAD fires END_OF_SPEECH, it already waited for the silence_duration
             this.speaking = false;
@@ -692,6 +768,10 @@ export class AudioRecognition {
         this.logger.debug('User turn committed');
       })
       .catch((err: unknown) => {
+        if (err instanceof Error && err.name === 'AbortError') {
+          this.logger.debug('User turn commit task cancelled');
+          return;
+        }
         this.logger.error(err, 'Error in user turn commit task:');
       });
   }