@livekit/agents 1.0.18 → 1.0.20

package/src/voice/agent_session.ts

@@ -14,7 +14,7 @@ import {
   type TTSModelString,
 } from '../inference/index.js';
 import { getJobContext } from '../job.js';
-import { ChatContext, ChatMessage } from '../llm/chat_context.js';
+import { AgentHandoffItem, ChatContext, ChatMessage } from '../llm/chat_context.js';
 import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import { log } from '../log.js';
@@ -26,6 +26,7 @@ import type { Agent } from './agent.js';
 import { AgentActivity } from './agent_activity.js';
 import type { _TurnDetector } from './audio_recognition.js';
 import {
+  type AgentEvent,
   AgentSessionEventTypes,
   type AgentState,
   type AgentStateChangedEvent,
@@ -127,6 +128,9 @@ export class AgentSession<
   private closingTask: Promise<void> | null = null;
   private userAwayTimer: NodeJS.Timeout | null = null;
 
+  /** @internal */
+  _recordedEvents: AgentEvent[] = [];
+
   constructor(opts: AgentSessionOptions<UserData>) {
     super();
 
@@ -174,6 +178,15 @@ export class AgentSession<
     this.on(AgentSessionEventTypes.UserInputTranscribed, this._onUserInputTranscribed.bind(this));
   }
 
+  emit<K extends keyof AgentSessionCallbacks>(
+    event: K,
+    ...args: Parameters<AgentSessionCallbacks[K]>
+  ): boolean {
+    const eventData = args[0] as AgentEvent;
+    this._recordedEvents.push(eventData);
+    return super.emit(event, ...args);
+  }
+
   get input(): AgentInput {
     return this._input;
   }
@@ -199,15 +212,20 @@ export class AgentSession<
   }
 
   async start({
+    // TODO(brian): PR2 - Add setupCloudTracer() call if on LiveKit Cloud with recording enabled
+    // TODO(brian): PR3 - Add span: this._sessionSpan = tracer.startSpan('agent_session'), store as instance property
+    // TODO(brian): PR4 - Add setupCloudLogger() call in setupCloudTracer() to setup OTEL logging with Pino bridge
     agent,
     room,
     inputOptions,
     outputOptions,
+    record = true,
   }: {
     agent: Agent;
     room: Room;
     inputOptions?: Partial<RoomInputOptions>;
     outputOptions?: Partial<RoomOutputOptions>;
+    record?: boolean;
   }): Promise<void> {
     if (this.started) {
       return;
@@ -247,6 +265,17 @@ export class AgentSession<
       this.logger.debug('Auto-connecting to room via job context');
       tasks.push(ctx.connect());
     }
+
+    if (record) {
+      if (ctx._primaryAgentSession === undefined) {
+        ctx._primaryAgentSession = this;
+      } else {
+        throw new Error(
+          'Only one `AgentSession` can be the primary at a time. If you want to ignore primary designation, use session.start(record=False).',
+        );
+      }
+    }
+
     // TODO(AJS-265): add shutdown callback to job context
     tasks.push(this.updateActivity(this.agent));
 
@@ -341,6 +370,8 @@ export class AgentSession<
     // TODO(AJS-129): add lock to agent activity core lifecycle
     this.nextActivity = new AgentActivity(agent, this);
 
+    const previousActivity = this.activity;
+
     if (this.activity) {
       await this.activity.drain();
       await this.activity.close();
@@ -349,6 +380,14 @@ export class AgentSession<
     this.activity = this.nextActivity;
     this.nextActivity = undefined;
 
+    this._chatCtx.insert(
+      new AgentHandoffItem({
+        oldAgentId: previousActivity?.agent.id,
+        newAgentId: agent.id,
+      }),
+    );
+    this.logger.debug({ previousActivity, agent }, 'Agent handoff inserted into chat context');
+
     await this.activity.start();
 
     if (this._input.audio) {
@@ -419,6 +458,8 @@ export class AgentSession<
       return;
     }
 
+    // TODO(brian): PR3 - Add span: if state === 'speaking' && !this._agentSpeakingSpan, create tracer.startSpan('agent_speaking') with participant attributes
+    // TODO(brian): PR3 - Add span: if state !== 'speaking' && this._agentSpeakingSpan, end and clear this._agentSpeakingSpan
     const oldState = this._agentState;
     this._agentState = state;
 
@@ -441,6 +482,8 @@ export class AgentSession<
       return;
     }
 
+    // TODO(brian): PR3 - Add span: if state === 'speaking' && !this._userSpeakingSpan, create tracer.startSpan('user_speaking') with participant attributes
+    // TODO(brian): PR3 - Add span: if state !== 'speaking' && this._userSpeakingSpan, end and clear this._userSpeakingSpan
     const oldState = this.userState;
     this.userState = state;
 

package/src/voice/audio_recognition.ts

@@ -57,6 +57,8 @@ export interface AudioRecognitionOptions {
   maxEndpointingDelay: number;
 }
 
+// TODO(brian): PR3 - Add span: private _userTurnSpan?: Span, create lazily in _ensureUserTurnSpan() method (tracer.startSpan('user_turn') with participant attributes)
+// TODO(brian): PR3 - Add span: 'eou_detection' span when running EOU detection (in runEOUDetection method)
 export class AudioRecognition {
   private hooks: RecognitionHooks;
   private stt?: STTNode;
@@ -356,7 +358,7 @@ export class AudioRecognition {
 
         if (turnDetector) {
           this.logger.debug('Running turn detector model');
-          if (!turnDetector.supportsLanguage(this.lastLanguage)) {
+          if (!(await turnDetector.supportsLanguage(this.lastLanguage))) {
             this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
           } else {
             const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);

package/src/voice/generation.ts

@@ -377,6 +377,7 @@ export function updateInstructions(options: {
   }
 }
 
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('llm_node') decorator/wrapper
 export function performLLMInference(
   node: LLMNode,
   chatCtx: ChatContext,
@@ -467,6 +468,7 @@ export function performLLMInference(
   ];
 }
 
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('tts_node') decorator/wrapper
 export function performTTSInference(
   node: TTSNode,
   text: ReadableStream<string>,
@@ -650,6 +652,7 @@ export function performAudioForwarding(
   ];
 }
 
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('function_tool') wrapper for each tool execution
 export function performToolExecutions({
   session,
   speechHandle,

package/src/voice/index.ts

@@ -6,5 +6,6 @@ export { AgentSession, type AgentSessionOptions } from './agent_session.js';
 export * from './avatar/index.js';
 export * from './background_audio.js';
 export * from './events.js';
+export * from './report.js';
 export * from './room_io/index.js';
 export { RunContext } from './run_context.js';

package/src/voice/report.ts

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext } from '../llm/chat_context.js';
+import type { VoiceOptions } from './agent_session.js';
+import type { AgentEvent } from './events.js';
+
+export interface SessionReport {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: VoiceOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableUserDataTraining: boolean;
+  timestamp: number;
+}
+
+export interface SessionReportOptions {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: VoiceOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableUserDataTraining?: boolean;
+  timestamp?: number;
+}
+
+export function createSessionReport(opts: SessionReportOptions): SessionReport {
+  return {
+    jobId: opts.jobId,
+    roomId: opts.roomId,
+    room: opts.room,
+    options: opts.options,
+    events: opts.events,
+    chatHistory: opts.chatHistory,
+    enableUserDataTraining: opts.enableUserDataTraining ?? false,
+    timestamp: opts.timestamp ?? Date.now(),
+  };
+}
+
+// TODO(brian): PR5 - Add uploadSessionReport() function that creates multipart form with:
+//   - header: protobuf MetricsRecordingHeader (room_id, duration, start_time)
+//   - chat_history: JSON serialized chat history (use sessionReportToJSON)
+//   - audio: audio recording file if available (ogg format)
+//   - Uploads to LiveKit Cloud observability endpoint with JWT auth
+export function sessionReportToJSON(report: SessionReport): Record<string, unknown> {
+  const events: Record<string, unknown>[] = [];
+
+  for (const event of report.events) {
+    if (event.type === 'metrics_collected') {
+      continue; // metrics are too noisy, Cloud is using the chat_history as the source of truth
+    }
+
+    events.push({ ...event });
+  }
+
+  return {
+    job_id: report.jobId,
+    room_id: report.roomId,
+    room: report.room,
+    events,
+    options: {
+      allow_interruptions: report.options.allowInterruptions,
+      discard_audio_if_uninterruptible: report.options.discardAudioIfUninterruptible,
+      min_interruption_duration: report.options.minInterruptionDuration,
+      min_interruption_words: report.options.minInterruptionWords,
+      min_endpointing_delay: report.options.minEndpointingDelay,
+      max_endpointing_delay: report.options.maxEndpointingDelay,
+      max_tool_steps: report.options.maxToolSteps,
+    },
+    chat_history: report.chatHistory.toJSON({ excludeTimestamp: false }),
+    enable_user_data_training: report.enableUserDataTraining,
+    timestamp: report.timestamp,
+  };
+}