@livekit/agents 1.1.0 → 1.2.0

package/src/voice/report.test.ts

@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { ChatContext } from '../llm/chat_context.js';
+import type { AgentSessionOptions, VoiceOptions } from './agent_session.js';
+import { createSessionReport, sessionReportToJSON } from './report.js';
+
+type ReportOptions = AgentSessionOptions & Partial<VoiceOptions>;
+
+function baseOptions(): ReportOptions {
+  return {
+    maxToolSteps: 3,
+    preemptiveGeneration: false,
+    userAwayTimeout: 15,
+    useTtsAlignedTranscript: true,
+    turnHandling: {},
+  };
+}
+
+function serializeOptions(options: ReportOptions) {
+  const report = createSessionReport({
+    jobId: 'job',
+    roomId: 'room-id',
+    room: 'room',
+    options,
+    events: [],
+    chatHistory: ChatContext.empty(),
+    enableRecording: false,
+    timestamp: 0,
+    startedAt: 0,
+  });
+
+  const payload = sessionReportToJSON(report);
+  return payload.options as Record<string, unknown>;
+}
+
+describe('sessionReportToJSON', () => {
+  it('serializes interruption and endpointing values from turnHandling', () => {
+    const options = baseOptions();
+    options.turnHandling = {
+      interruption: {
+        mode: 'adaptive',
+        discardAudioIfUninterruptible: false,
+        minDuration: 1200,
+        minWords: 2,
+      },
+      endpointing: {
+        minDelay: 900,
+        maxDelay: 4500,
+      },
+    };
+
+    const serialized = serializeOptions(options);
+    expect(serialized).toMatchObject({
+      allow_interruptions: true,
+      discard_audio_if_uninterruptible: false,
+      min_interruption_duration: 1200,
+      min_interruption_words: 2,
+      min_endpointing_delay: 900,
+      max_endpointing_delay: 4500,
+      max_tool_steps: 3,
+    });
+  });
+
+  it('prefers turnHandling values over deprecated flat fields', () => {
+    const options = baseOptions();
+    options.allowInterruptions = false;
+    options.discardAudioIfUninterruptible = true;
+    options.minInterruptionDuration = 400;
+    options.minInterruptionWords = 1;
+    options.minEndpointingDelay = 500;
+    options.maxEndpointingDelay = 2500;
+    options.turnHandling = {
+      interruption: {
+        mode: 'vad',
+        discardAudioIfUninterruptible: false,
+        minDuration: 1400,
+        minWords: 4,
+      },
+      endpointing: {
+        minDelay: 700,
+        maxDelay: 3900,
+      },
+    };
+
+    const serialized = serializeOptions(options);
+    expect(serialized).toMatchObject({
+      allow_interruptions: true,
+      discard_audio_if_uninterruptible: false,
+      min_interruption_duration: 1400,
+      min_interruption_words: 4,
+      min_endpointing_delay: 700,
+      max_endpointing_delay: 3900,
+      max_tool_steps: 3,
+    });
+  });
+
+  it('serializes allow_interruptions from interruption.enabled when present', () => {
+    const options = baseOptions();
+    options.allowInterruptions = true;
+    options.turnHandling = {
+      interruption: {
+        enabled: false,
+        mode: 'adaptive',
+      },
+    };
+
+    const serialized = serializeOptions(options);
+    expect(serialized).toMatchObject({
+      allow_interruptions: false,
+      max_tool_steps: 3,
+    });
+  });
+
+  it('falls back to deprecated flat fields when turnHandling values are absent', () => {
+    const options = baseOptions();
+    options.allowInterruptions = false;
+    options.discardAudioIfUninterruptible = false;
+    options.minInterruptionDuration = 600;
+    options.minInterruptionWords = 3;
+    options.minEndpointingDelay = 1000;
+    options.maxEndpointingDelay = 5000;
+
+    const serialized = serializeOptions(options);
+    expect(serialized).toMatchObject({
+      allow_interruptions: false,
+      discard_audio_if_uninterruptible: false,
+      min_interruption_duration: 600,
+      min_interruption_words: 3,
+      min_endpointing_delay: 1000,
+      max_endpointing_delay: 5000,
+      max_tool_steps: 3,
+    });
+  });
+});

package/src/voice/report.ts

@@ -0,0 +1,140 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext } from '../llm/chat_context.js';
+import { type ModelUsage, filterZeroValues } from '../metrics/model_usage.js';
+import type { AgentSessionOptions, VoiceOptions } from './agent_session.js';
+import type { AgentEvent } from './events.js';
+
+type ReportOptions = AgentSessionOptions & Partial<VoiceOptions>;
+
+export interface SessionReport {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: ReportOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableRecording: boolean;
+  /** Timestamp when the session started (milliseconds) */
+  startedAt: number;
+  /** Timestamp when the session report was created (milliseconds), typically at the end of the session */
+  timestamp: number;
+  /** Path to the audio recording file (if recording was enabled) */
+  audioRecordingPath?: string;
+  /** Timestamp when the audio recording started (milliseconds) */
+  audioRecordingStartedAt?: number;
+  /** Duration of the session in milliseconds */
+  duration?: number;
+  /** Usage summaries for the session, one per model/provider combination */
+  modelUsage?: ModelUsage[];
+}
+
+export interface SessionReportOptions {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: ReportOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableRecording?: boolean;
+  /** Timestamp when the session started (milliseconds) */
+  startedAt?: number;
+  /** Timestamp when the session report was created (milliseconds) */
+  timestamp?: number;
+  /** Path to the audio recording file (if recording was enabled) */
+  audioRecordingPath?: string;
+  /** Timestamp when the audio recording started (milliseconds) */
+  audioRecordingStartedAt?: number;
+  /** Usage summaries for the session, one per model/provider combination */
+  modelUsage?: ModelUsage[];
+}
+
+export function createSessionReport(opts: SessionReportOptions): SessionReport {
+  const timestamp = opts.timestamp ?? Date.now();
+  const audioRecordingStartedAt = opts.audioRecordingStartedAt;
+
+  return {
+    jobId: opts.jobId,
+    roomId: opts.roomId,
+    room: opts.room,
+    options: opts.options,
+    events: opts.events,
+    chatHistory: opts.chatHistory,
+    enableRecording: opts.enableRecording ?? false,
+    startedAt: opts.startedAt ?? Date.now(),
+    timestamp,
+    audioRecordingPath: opts.audioRecordingPath,
+    audioRecordingStartedAt,
+    duration:
+      audioRecordingStartedAt !== undefined ? timestamp - audioRecordingStartedAt : undefined,
+    modelUsage: opts.modelUsage,
+  };
+}
+
+//   - header: protobuf MetricsRecordingHeader (room_id, duration, start_time)
+//   - chat_history: JSON serialized chat history (use sessionReportToJSON)
+//   - audio: audio recording file if available (ogg format)
+//   - Uploads to LiveKit Cloud observability endpoint with JWT auth
+export function sessionReportToJSON(report: SessionReport): Record<string, unknown> {
+  const events: Record<string, unknown>[] = [];
+  const options = report.options;
+  const interruptionConfig = options.turnHandling?.interruption;
+  const endpointingConfig = options.turnHandling?.endpointing;
+
+  // Keep backwards compatibility with deprecated fields
+  const allowInterruptions =
+    interruptionConfig?.enabled !== undefined
+      ? interruptionConfig.enabled
+      : interruptionConfig?.mode !== undefined
+        ? true
+        : options.allowInterruptions ?? options.voiceOptions?.allowInterruptions;
+  const discardAudioIfUninterruptible =
+    interruptionConfig?.discardAudioIfUninterruptible ??
+    options.discardAudioIfUninterruptible ??
+    options.voiceOptions?.discardAudioIfUninterruptible;
+  const minInterruptionDuration =
+    interruptionConfig?.minDuration ??
+    options.minInterruptionDuration ??
+    options.voiceOptions?.minInterruptionDuration;
+  const minInterruptionWords =
+    interruptionConfig?.minWords ??
+    options.minInterruptionWords ??
+    options.voiceOptions?.minInterruptionWords;
+  const minEndpointingDelay =
+    endpointingConfig?.minDelay ??
+    options.minEndpointingDelay ??
+    options.voiceOptions?.minEndpointingDelay;
+  const maxEndpointingDelay =
+    endpointingConfig?.maxDelay ??
+    options.maxEndpointingDelay ??
+    options.voiceOptions?.maxEndpointingDelay;
+
+  for (const event of report.events) {
+    if (event.type === 'metrics_collected') {
+      continue; // metrics are too noisy, Cloud is using the chat_history as the source of truth
+    }
+
+    events.push({ ...event });
+  }
+
+  return {
+    job_id: report.jobId,
+    room_id: report.roomId,
+    room: report.room,
+    events,
+    options: {
+      allow_interruptions: allowInterruptions,
+      discard_audio_if_uninterruptible: discardAudioIfUninterruptible,
+      min_interruption_duration: minInterruptionDuration,
+      min_interruption_words: minInterruptionWords,
+      min_endpointing_delay: minEndpointingDelay,
+      max_endpointing_delay: maxEndpointingDelay,
+      max_tool_steps: options.maxToolSteps,
+    },
+    chat_history: report.chatHistory.toJSON({ excludeTimestamp: false }),
+    enable_user_data_training: report.enableRecording,
+    timestamp: report.timestamp,
+    usage: report.modelUsage ? report.modelUsage.map(filterZeroValues) : null,
+  };
+}

package/src/voice/room_io/_input.ts

@@ -2,8 +2,9 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import {
-  AudioFrame,
+  type AudioFrame,
   AudioStream,
+  FrameProcessor,
   type NoiseCancellationOptions,
   RemoteParticipant,
   type RemoteTrack,
@@ -22,9 +23,12 @@ export class ParticipantAudioInputStream extends AudioInput {
   private sampleRate: number;
   private numChannels: number;
   private noiseCancellation?: NoiseCancellationOptions;
+  private frameProcessor?: FrameProcessor<AudioFrame>;
   private publication: RemoteTrackPublication | null = null;
   private participantIdentity: string | null = null;
+  private currentInputId: string | null = null;
   private logger = log();
+
   constructor({
     room,
     sampleRate,
@@ -34,16 +38,21 @@ export class ParticipantAudioInputStream extends AudioInput {
     room: Room;
     sampleRate: number;
     numChannels: number;
-    noiseCancellation?: NoiseCancellationOptions;
+    noiseCancellation?: NoiseCancellationOptions | FrameProcessor<AudioFrame>;
   }) {
     super();
     this.room = room;
     this.sampleRate = sampleRate;
     this.numChannels = numChannels;
-    this.noiseCancellation = noiseCancellation;
+    if (noiseCancellation instanceof FrameProcessor) {
+      this.frameProcessor = noiseCancellation;
+    } else {
+      this.noiseCancellation = noiseCancellation;
+    }
 
     this.room.on(RoomEvent.TrackSubscribed, this.onTrackSubscribed);
     this.room.on(RoomEvent.TrackUnpublished, this.onTrackUnpublished);
+    this.room.on(RoomEvent.TokenRefreshed, this.onTokenRefreshed);
   }
 
   setParticipant(participant: RemoteParticipant | string | null) {
@@ -54,8 +63,10 @@ export class ParticipantAudioInputStream extends AudioInput {
     if (this.participantIdentity === participantIdentity) {
       return;
     }
+    if (this.participantIdentity) {
+      this.closeStream();
+    }
     this.participantIdentity = participantIdentity;
-    this.closeStream();
 
     if (!participantIdentity) {
       return;
@@ -113,9 +124,11 @@ export class ParticipantAudioInputStream extends AudioInput {
   };
 
   private closeStream() {
-    if (this.deferredStream.isSourceSet) {
-      this.deferredStream.detachSource();
+    if (this.currentInputId) {
+      void this.multiStream.removeInputStream(this.currentInputId);
+      this.currentInputId = null;
     }
+
     this.publication = null;
   }
 
@@ -134,28 +147,50 @@ export class ParticipantAudioInputStream extends AudioInput {
     }
     this.closeStream();
     this.publication = publication;
-    this.deferredStream.setSource(
+    this.currentInputId = this.multiStream.addInputStream(
       resampleStream({
         stream: this.createStream(track),
         outputRate: this.sampleRate,
       }),
     );
+    this.frameProcessor?.onStreamInfoUpdated({
+      participantIdentity: participant.identity,
+      roomName: this.room.name!,
+      publicationSid: publication.sid!,
+    });
+    this.frameProcessor?.onCredentialsUpdated({
+      token: this.room.token!,
+      url: this.room.serverUrl!,
+    });
     return true;
   };
 
+  private onTokenRefreshed = () => {
+    if (this.room.token && this.room.serverUrl) {
+      this.frameProcessor?.onCredentialsUpdated({
+        token: this.room.token,
+        url: this.room.serverUrl,
+      });
+    }
+  };
+
   private createStream(track: RemoteTrack): ReadableStream<AudioFrame> {
     return new AudioStream(track, {
       sampleRate: this.sampleRate,
       numChannels: this.numChannels,
-      noiseCancellation: this.noiseCancellation,
+      noiseCancellation: this.frameProcessor || this.noiseCancellation,
       // TODO(AJS-269): resolve compatibility issue with node-sdk to remove the forced type casting
     }) as unknown as ReadableStream<AudioFrame>;
   }
 
-  async close() {
+  override async close() {
     this.room.off(RoomEvent.TrackSubscribed, this.onTrackSubscribed);
     this.room.off(RoomEvent.TrackUnpublished, this.onTrackUnpublished);
+    this.room.off(RoomEvent.TokenRefreshed, this.onTokenRefreshed);
     this.closeStream();
-    this.deferredStream.stream.cancel();
+    await super.close();
+
+    this.frameProcessor?.close();
+    this.frameProcessor = undefined;
   }
 }

package/src/voice/room_io/_output.ts

@@ -23,7 +23,7 @@ import {
 } from '../../constants.js';
 import { log } from '../../log.js';
 import { Future, Task, shortuuid } from '../../utils.js';
-import { AudioOutput, TextOutput } from '../io.js';
+import { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js';
 import { findMicrophoneTrackId } from '../transcription/index.js';
 
 abstract class BaseParticipantTranscriptionOutput extends TextOutput {
@@ -102,13 +102,14 @@ abstract class BaseParticipantTranscriptionOutput extends TextOutput {
     this.latestText = '';
   }
 
-  async captureText(text: string) {
+  async captureText(text: string | TimedString) {
     if (!this.participantIdentity) {
       return;
     }
 
-    this.latestText = text;
-    await this.handleCaptureText(text);
+    const textStr = isTimedString(text) ? text.text : text;
+    this.latestText = textStr;
+    await this.handleCaptureText(textStr);
   }
 
   flush() {
@@ -290,7 +291,7 @@ export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscr
 }
 
 export class ParalellTextOutput extends TextOutput {
-  /* @internal */
+  /** @internal */
   _sinks: TextOutput[];
 
   constructor(sinks: TextOutput[], nextInChain?: TextOutput) {
@@ -298,7 +299,7 @@ export class ParalellTextOutput extends TextOutput {
     this._sinks = sinks;
   }
 
-  async captureText(text: string) {
+  async captureText(text: string | TimedString) {
     await Promise.all(this._sinks.map((sink) => sink.captureText(text)));
   }
 
@@ -321,12 +322,15 @@ export class ParticipantAudioOutput extends AudioOutput {
   private audioSource: AudioSource;
   private publication?: LocalTrackPublication;
   private flushTask?: Task<void>;
-  private pushedDurationMs: number = 0;
+
+  /** Duration of audio pushed to the source, in seconds */
+  private pushedDuration: number = 0;
   private startedFuture: Future<void> = new Future();
   private interruptedFuture: Future<void> = new Future();
+  private firstFrameEmitted: boolean = false;
 
   constructor(room: Room, options: AudioOutputOptions) {
-    super(options.sampleRate);
+    super(options.sampleRate, undefined, { pause: true });
     this.room = room;
     this.options = options;
     this.audioSource = new AudioSource(options.sampleRate, options.numChannels);
@@ -345,8 +349,13 @@ export class ParticipantAudioOutput extends AudioOutput {
 
     super.captureFrame(frame);
 
+    if (!this.firstFrameEmitted) {
+      this.firstFrameEmitted = true;
+      this.onPlaybackStarted(Date.now());
+    }
+
     // TODO(AJS-102): use frame.durationMs once available in rtc-node
-    this.pushedDurationMs += frame.samplesPerChannel / frame.sampleRate;
+    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;
     await this.audioSource.captureFrame(frame);
   }
 
@@ -369,16 +378,19 @@ export class ParticipantAudioOutput extends AudioOutput {
       this.interruptedFuture.await.then(() => true),
     ]);
 
-    let pushedDuration = this.pushedDurationMs;
+    let pushedDuration = this.pushedDuration;
 
     if (interrupted) {
       // Calculate actual played duration accounting for queued audio
-      pushedDuration = Math.max(this.pushedDurationMs - this.audioSource.queuedDuration, 0);
+      // Note: queuedDuration is in milliseconds, pushedDuration is in seconds
+      pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);
       this.audioSource.clearQueue();
     }
 
-    this.pushedDurationMs = 0;
+    this.pushedDuration = 0;
     this.interruptedFuture = new Future();
+    this.firstFrameEmitted = false;
+
     this.onPlaybackFinished({
       playbackPosition: pushedDuration,
       interrupted,
@@ -391,7 +403,7 @@ export class ParticipantAudioOutput extends AudioOutput {
   flush(): void {
     super.flush();
 
-    if (!this.pushedDurationMs) {
+    if (!this.pushedDuration) {
       return;
     }
 
@@ -404,7 +416,7 @@ export class ParticipantAudioOutput extends AudioOutput {
   }
 
   clearBuffer(): void {
-    if (!this.pushedDurationMs) {
+    if (!this.pushedDuration) {
       return;
     }