@livekit/agents 1.0.17 → 1.0.19

package/src/voice/agent_session.ts

@@ -14,7 +14,7 @@ import {
   type TTSModelString,
 } from '../inference/index.js';
 import { getJobContext } from '../job.js';
-import { ChatContext, ChatMessage } from '../llm/chat_context.js';
+import { AgentHandoffItem, ChatContext, ChatMessage } from '../llm/chat_context.js';
 import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import { log } from '../log.js';
@@ -26,6 +26,7 @@ import type { Agent } from './agent.js';
 import { AgentActivity } from './agent_activity.js';
 import type { _TurnDetector } from './audio_recognition.js';
 import {
+  type AgentEvent,
   AgentSessionEventTypes,
   type AgentState,
   type AgentStateChangedEvent,
@@ -58,6 +59,7 @@ export interface VoiceOptions {
   maxEndpointingDelay: number;
   maxToolSteps: number;
   preemptiveGeneration: boolean;
+  userAwayTimeout?: number | null;
 }
 
 const defaultVoiceOptions: VoiceOptions = {
@@ -69,6 +71,7 @@ const defaultVoiceOptions: VoiceOptions = {
   maxEndpointingDelay: 6000,
   maxToolSteps: 3,
   preemptiveGeneration: false,
+  userAwayTimeout: 15.0,
 } as const;
 
 export type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;
@@ -123,6 +126,10 @@ export class AgentSession<
   private _output: AgentOutput;
 
   private closingTask: Promise<void> | null = null;
+  private userAwayTimer: NodeJS.Timeout | null = null;
+
+  /** @internal */
+  _recordedEvents: AgentEvent[] = [];
 
   constructor(opts: AgentSessionOptions<UserData>) {
     super();
@@ -167,6 +174,17 @@ export class AgentSession<
     // This is the "global" chat context, it holds the entire conversation history
     this._chatCtx = ChatContext.empty();
     this.options = { ...defaultVoiceOptions, ...voiceOptions };
+
+    this.on(AgentSessionEventTypes.UserInputTranscribed, this._onUserInputTranscribed.bind(this));
+  }
+
+  emit<K extends keyof AgentSessionCallbacks>(
+    event: K,
+    ...args: Parameters<AgentSessionCallbacks[K]>
+  ): boolean {
+    const eventData = args[0] as AgentEvent;
+    this._recordedEvents.push(eventData);
+    return super.emit(event, ...args);
   }
 
   get input(): AgentInput {
@@ -194,15 +212,20 @@ export class AgentSession<
   }
 
   async start({
+    // TODO(brian): PR2 - Add setupCloudTracer() call if on LiveKit Cloud with recording enabled
+    // TODO(brian): PR3 - Add span: this._sessionSpan = tracer.startSpan('agent_session'), store as instance property
+    // TODO(brian): PR4 - Add setupCloudLogger() call in setupCloudTracer() to setup OTEL logging with Pino bridge
     agent,
     room,
     inputOptions,
     outputOptions,
+    record = true,
   }: {
     agent: Agent;
     room: Room;
     inputOptions?: Partial<RoomInputOptions>;
     outputOptions?: Partial<RoomOutputOptions>;
+    record?: boolean;
   }): Promise<void> {
     if (this.started) {
       return;
@@ -242,6 +265,17 @@ export class AgentSession<
       this.logger.debug('Auto-connecting to room via job context');
       tasks.push(ctx.connect());
     }
+
+    if (record) {
+      if (ctx._primaryAgentSession === undefined) {
+        ctx._primaryAgentSession = this;
+      } else {
+        throw new Error(
+          'Only one `AgentSession` can be the primary at a time. If you want to ignore primary designation, use session.start(record=False).',
+        );
+      }
+    }
+
     // TODO(AJS-265): add shutdown callback to job context
     tasks.push(this.updateActivity(this.agent));
 
@@ -336,6 +370,8 @@ export class AgentSession<
     // TODO(AJS-129): add lock to agent activity core lifecycle
     this.nextActivity = new AgentActivity(agent, this);
 
+    const previousActivity = this.activity;
+
     if (this.activity) {
       await this.activity.drain();
       await this.activity.close();
@@ -344,6 +380,14 @@ export class AgentSession<
     this.activity = this.nextActivity;
     this.nextActivity = undefined;
 
+    this._chatCtx.insert(
+      new AgentHandoffItem({
+        oldAgentId: previousActivity?.agent.id,
+        newAgentId: agent.id,
+      }),
+    );
+    this.logger.debug({ previousActivity, agent }, 'Agent handoff inserted into chat context');
+
     await this.activity.start();
 
     if (this._input.audio) {
@@ -414,8 +458,18 @@ export class AgentSession<
       return;
     }
 
+    // TODO(brian): PR3 - Add span: if state === 'speaking' && !this._agentSpeakingSpan, create tracer.startSpan('agent_speaking') with participant attributes
+    // TODO(brian): PR3 - Add span: if state !== 'speaking' && this._agentSpeakingSpan, end and clear this._agentSpeakingSpan
     const oldState = this._agentState;
     this._agentState = state;
+
+    // Handle user away timer based on state changes
+    if (state === 'listening' && this.userState === 'listening') {
+      this._setUserAwayTimer();
+    } else {
+      this._cancelUserAwayTimer();
+    }
+
     this.emit(
       AgentSessionEventTypes.AgentStateChanged,
       createAgentStateChangedEvent(oldState, state),
@@ -428,8 +482,18 @@ export class AgentSession<
       return;
     }
 
+    // TODO(brian): PR3 - Add span: if state === 'speaking' && !this._userSpeakingSpan, create tracer.startSpan('user_speaking') with participant attributes
+    // TODO(brian): PR3 - Add span: if state !== 'speaking' && this._userSpeakingSpan, end and clear this._userSpeakingSpan
     const oldState = this.userState;
     this.userState = state;
+
+    // Handle user away timer based on state changes
+    if (state === 'listening' && this._agentState === 'listening') {
+      this._setUserAwayTimer();
+    } else {
+      this._cancelUserAwayTimer();
+    }
+
     this.emit(
       AgentSessionEventTypes.UserStateChanged,
       createUserStateChangedEvent(oldState, state),
@@ -451,6 +515,37 @@ export class AgentSession<
 
   private onTextOutputChanged(): void {}
 
+  private _setUserAwayTimer(): void {
+    this._cancelUserAwayTimer();
+
+    if (this.options.userAwayTimeout === null || this.options.userAwayTimeout === undefined) {
+      return;
+    }
+
+    if (this.roomIO && !this.roomIO.isParticipantAvailable) {
+      return;
+    }
+
+    this.userAwayTimer = setTimeout(() => {
+      this.logger.debug('User away timeout triggered');
+      this._updateUserState('away');
+    }, this.options.userAwayTimeout * 1000);
+  }
+
+  private _cancelUserAwayTimer(): void {
+    if (this.userAwayTimer !== null) {
+      clearTimeout(this.userAwayTimer);
+      this.userAwayTimer = null;
+    }
+  }
+
+  private _onUserInputTranscribed(ev: UserInputTranscribedEvent): void {
+    if (this.userState === 'away' && ev.isFinal) {
+      this.logger.debug('User returned from away state due to speech input');
+      this._updateUserState('listening');
+    }
+  }
+
   private async closeImpl(
     reason: CloseReason,
     error: RealtimeModelError | LLMError | TTSError | STTError | null = null,
@@ -460,6 +555,8 @@ export class AgentSession<
       return;
     }
 
+    this._cancelUserAwayTimer();
+
     if (this.activity) {
       if (!drain) {
         try {

package/src/voice/audio_recognition.ts

@@ -57,6 +57,8 @@ export interface AudioRecognitionOptions {
   maxEndpointingDelay: number;
 }
 
+// TODO(brian): PR3 - Add span: private _userTurnSpan?: Span, create lazily in _ensureUserTurnSpan() method (tracer.startSpan('user_turn') with participant attributes)
+// TODO(brian): PR3 - Add span: 'eou_detection' span when running EOU detection (in runEOUDetection method)
 export class AudioRecognition {
   private hooks: RecognitionHooks;
   private stt?: STTNode;

package/src/voice/generation.ts

@@ -377,6 +377,7 @@ export function updateInstructions(options: {
   }
 }
 
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('llm_node') decorator/wrapper
 export function performLLMInference(
   node: LLMNode,
   chatCtx: ChatContext,
@@ -467,6 +468,7 @@ export function performLLMInference(
   ];
 }
 
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('tts_node') decorator/wrapper
 export function performTTSInference(
   node: TTSNode,
   text: ReadableStream<string>,
@@ -650,6 +652,7 @@ export function performAudioForwarding(
   ];
 }
 
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('function_tool') wrapper for each tool execution
 export function performToolExecutions({
   session,
   speechHandle,

package/src/voice/index.ts

@@ -6,5 +6,6 @@ export { AgentSession, type AgentSessionOptions } from './agent_session.js';
 export * from './avatar/index.js';
 export * from './background_audio.js';
 export * from './events.js';
+export * from './report.js';
 export * from './room_io/index.js';
 export { RunContext } from './run_context.js';

package/src/voice/interruption_detection.test.ts

@@ -0,0 +1,151 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Unit tests for interruption detection logic in AgentActivity.
+ *
+ * Tests the refactored minInterruptionWords check which ensures:
+ * - Consistent word count filtering across all speech scenarios
+ * - Proper handling of empty strings, undefined, and short speech
+ * - Interruptions allowed only when word count meets or exceeds minInterruptionWords threshold
+ */
+import { describe, expect, it } from 'vitest';
+import { splitWords } from '../tokenize/basic/word.js';
+
+describe('Interruption Detection - Word Counting', () => {
+  describe('Word Splitting Behavior', () => {
+    it('should count empty string as 0 words', () => {
+      const text = '';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(0);
+    });
+
+    it('should count single word correctly', () => {
+      const text = 'hello';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(1);
+    });
+
+    it('should count two words correctly', () => {
+      const text = 'hello world';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+
+    it('should count multiple words correctly', () => {
+      const text = 'hello this is a full sentence';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(6);
+    });
+
+    it('should handle punctuation correctly', () => {
+      const text = 'hello, world!';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+
+    it('should handle multiple spaces between words', () => {
+      const text = 'hello  world';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+
+    it('should count whitespace-only string as 0 words', () => {
+      const text = '   ';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(0);
+    });
+
+    it('should handle leading and trailing whitespace', () => {
+      const text = '  hello world  ';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+  });
+
+  describe('Integration: Full Interruption Check Logic', () => {
+    it('should block interruption for empty transcript with threshold 2', () => {
+      const text = '';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('');
+      expect(wordCount).toBe(0);
+      expect(shouldBlock).toBe(true);
+    });
+
+    it('should block interruption for undefined transcript with threshold 2', () => {
+      const text: string | undefined = undefined;
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('');
+      expect(wordCount).toBe(0);
+      expect(shouldBlock).toBe(true);
+    });
+
+    it('should block interruption for single word with threshold 2', () => {
+      const text = 'hello';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('hello');
+      expect(wordCount).toBe(1);
+      expect(shouldBlock).toBe(true);
+    });
+
+    it('should allow interruption when word count exactly meets threshold', () => {
+      const text = 'hello world';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('hello world');
+      expect(wordCount).toBe(2);
+      expect(shouldBlock).toBe(false);
+    });
+
+    it('should allow interruption when word count exceeds threshold', () => {
+      const text = 'hello this is a full sentence';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('hello this is a full sentence');
+      expect(wordCount).toBe(6);
+      expect(shouldBlock).toBe(false);
+    });
+
+    it('should apply consistent word counting logic in both methods', () => {
+      const transcripts = ['', 'hello', 'hello world', 'this is a longer sentence'];
+      const threshold = 2;
+
+      transcripts.forEach((transcript) => {
+        const text1 = transcript;
+        const normalizedText1 = text1 ?? '';
+        const wordCount1 = splitWords(normalizedText1, true).length;
+        const shouldBlock1 = wordCount1 < threshold;
+
+        const wordCount2 = splitWords(transcript, true).length;
+        const shouldBlock2 = wordCount2 < threshold;
+
+        expect(wordCount1).toBe(wordCount2);
+        expect(shouldBlock1).toBe(shouldBlock2);
+      });
+    });
+  });
+});

package/src/voice/report.ts

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext } from '../llm/chat_context.js';
+import type { VoiceOptions } from './agent_session.js';
+import type { AgentEvent } from './events.js';
+
+export interface SessionReport {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: VoiceOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableUserDataTraining: boolean;
+  timestamp: number;
+}
+
+export interface SessionReportOptions {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: VoiceOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableUserDataTraining?: boolean;
+  timestamp?: number;
+}
+
+export function createSessionReport(opts: SessionReportOptions): SessionReport {
+  return {
+    jobId: opts.jobId,
+    roomId: opts.roomId,
+    room: opts.room,
+    options: opts.options,
+    events: opts.events,
+    chatHistory: opts.chatHistory,
+    enableUserDataTraining: opts.enableUserDataTraining ?? false,
+    timestamp: opts.timestamp ?? Date.now(),
+  };
+}
+
+// TODO(brian): PR5 - Add uploadSessionReport() function that creates multipart form with:
+//   - header: protobuf MetricsRecordingHeader (room_id, duration, start_time)
+//   - chat_history: JSON serialized chat history (use sessionReportToJSON)
+//   - audio: audio recording file if available (ogg format)
+//   - Uploads to LiveKit Cloud observability endpoint with JWT auth
+export function sessionReportToJSON(report: SessionReport): Record<string, unknown> {
+  const events: Record<string, unknown>[] = [];
+
+  for (const event of report.events) {
+    if (event.type === 'metrics_collected') {
+      continue; // metrics are too noisy, Cloud is using the chat_history as the source of truth
+    }
+
+    events.push({ ...event });
+  }
+
+  return {
+    job_id: report.jobId,
+    room_id: report.roomId,
+    room: report.room,
+    events,
+    options: {
+      allow_interruptions: report.options.allowInterruptions,
+      discard_audio_if_uninterruptible: report.options.discardAudioIfUninterruptible,
+      min_interruption_duration: report.options.minInterruptionDuration,
+      min_interruption_words: report.options.minInterruptionWords,
+      min_endpointing_delay: report.options.minEndpointingDelay,
+      max_endpointing_delay: report.options.maxEndpointingDelay,
+      max_tool_steps: report.options.maxToolSteps,
+    },
+    chat_history: report.chatHistory.toJSON({ excludeTimestamp: false }),
+    enable_user_data_training: report.enableUserDataTraining,
+    timestamp: report.timestamp,
+  };
+}

package/src/voice/room_io/room_io.ts

@@ -369,6 +369,10 @@ export class RoomIO {
     return this.transcriptionSynchronizer.textOutput;
   }
 
+  get isParticipantAvailable(): boolean {
+    return this.participantAvailableFuture.done;
+  }
+
   /** Switch to a different participant */
   setParticipant(participantIdentity: string | null) {
     this.logger.debug({ participantIdentity }, 'setting participant');