@livekit/agents 1.0.47 → 1.1.0-dev.0

package/src/voice/events.ts

@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { InterruptionDetectionError } from '../inference/interruption/errors.js';
+import type { OverlappingSpeechEvent } from '../inference/interruption/types.js';
 import type {
   ChatMessage,
   FunctionCall,
@@ -25,6 +27,7 @@ export enum AgentSessionEventTypes {
   FunctionToolsExecuted = 'function_tools_executed',
   MetricsCollected = 'metrics_collected',
   SpeechCreated = 'speech_created',
+  UserOverlappingSpeech = 'user_overlapping_speech',
   Error = 'error',
   Close = 'close',
 }
@@ -215,13 +218,13 @@ export const createSpeechCreatedEvent = ({
 
 export type ErrorEvent = {
   type: 'error';
-  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;
+  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown;
   source: LLM | STT | TTS | RealtimeModel | unknown;
   createdAt: number;
 };
 
 export const createErrorEvent = (
-  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,
+  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown,
   source: LLM | STT | TTS | RealtimeModel | unknown,
   createdAt: number = Date.now(),
 ): ErrorEvent => ({
@@ -233,14 +236,20 @@ export const createErrorEvent = (
 
 export type CloseEvent = {
   type: 'close';
-  error: RealtimeModelError | STTError | TTSError | LLMError | null;
+  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null;
   reason: ShutdownReason;
   createdAt: number;
 };
 
 export const createCloseEvent = (
   reason: ShutdownReason,
-  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,
+  error:
+    | RealtimeModelError
+    | STTError
+    | TTSError
+    | LLMError
+    | InterruptionDetectionError
+    | null = null,
   createdAt: number = Date.now(),
 ): CloseEvent => ({
   type: 'close',
@@ -257,5 +266,6 @@ export type AgentEvent =
   | ConversationItemAddedEvent
   | FunctionToolsExecutedEvent
   | SpeechCreatedEvent
+  | OverlappingSpeechEvent
   | ErrorEvent
   | CloseEvent;

package/src/voice/generation.ts

@@ -51,6 +51,7 @@ export class _LLMGenerationData {
   generatedText: string = '';
   generatedToolCalls: FunctionCall[];
   id: string;
+  ttft?: number;
 
   constructor(
     public readonly textStream: ReadableStream<string>,
@@ -416,6 +417,8 @@ export function performLLMInference(
   toolCtx: ToolContext,
   modelSettings: ModelSettings,
   controller: AbortController,
+  model?: string,
+  provider?: string,
 ): [Task<void>, _LLMGenerationData] {
   const textStream = new IdentityTransform<string>();
   const toolCallStream = new IdentityTransform<FunctionCall>();
@@ -431,8 +434,17 @@ export function performLLMInference(
     );
     span.setAttribute(traceTypes.ATTR_FUNCTION_TOOLS, JSON.stringify(Object.keys(toolCtx)));
 
+    if (model) {
+      span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, model);
+    }
+    if (provider) {
+      span.setAttribute(traceTypes.ATTR_GEN_AI_PROVIDER_NAME, provider);
+    }
+
     let llmStreamReader: ReadableStreamDefaultReader<string | ChatChunk> | null = null;
     let llmStream: ReadableStream<string | ChatChunk> | null = null;
+    const startTime = performance.now() / 1000; // Convert to seconds
+    let firstTokenReceived = false;
 
     try {
       llmStream = await node(chatCtx, toolCtx, modelSettings);
@@ -455,6 +467,11 @@ export function performLLMInference(
         const { done, value: chunk } = result;
         if (done) break;
 
+        if (!firstTokenReceived) {
+          firstTokenReceived = true;
+          data.ttft = performance.now() / 1000 - startTime;
+        }
+
         if (typeof chunk === 'string') {
           data.generatedText += chunk;
           await textWriter.write(chunk);
@@ -493,6 +510,9 @@ export function performLLMInference(
       }
 
       span.setAttribute(traceTypes.ATTR_RESPONSE_TEXT, data.generatedText);
+      if (data.ttft !== undefined) {
+        span.setAttribute(traceTypes.ATTR_RESPONSE_TTFT, data.ttft);
+      }
     } catch (error) {
       if (error instanceof DOMException && error.name === 'AbortError') {
         // Abort signal was triggered, handle gracefully
@@ -527,6 +547,8 @@ export function performTTSInference(
   text: ReadableStream<string | TimedString>,
   modelSettings: ModelSettings,
   controller: AbortController,
+  model?: string,
+  provider?: string,
 ): [Task<void>, _TTSGenerationData] {
   const audioStream = new IdentityTransform<AudioFrame>();
   const outputWriter = audioStream.writable.getWriter();
@@ -558,10 +580,27 @@ export function performTTSInference(
     }
   })();
 
-  const _performTTSInferenceImpl = async (signal: AbortSignal) => {
+  let ttfb: number | undefined;
+
+  const genData: _TTSGenerationData = {
+    audioStream: audioOutputStream,
+    timedTextsFut,
+    ttfb: undefined,
+  };
+
+  const _performTTSInferenceImpl = async (signal: AbortSignal, span: Span) => {
+    if (model) {
+      span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, model);
+    }
+    if (provider) {
+      span.setAttribute(traceTypes.ATTR_GEN_AI_PROVIDER_NAME, provider);
+    }
+
     let ttsStreamReader: ReadableStreamDefaultReader<AudioFrame> | null = null;
     let ttsStream: ReadableStream<AudioFrame> | null = null;
     let pushedDuration = 0;
+    const startTime = performance.now() / 1000; // Convert to seconds
+    let firstByteReceived = false;
 
     try {
       ttsStream = await node(textOnlyStream.readable, modelSettings);
@@ -595,6 +634,13 @@ export function performTTSInference(
           break;
         }
 
+        if (!firstByteReceived) {
+          firstByteReceived = true;
+          ttfb = performance.now() / 1000 - startTime;
+          genData.ttfb = ttfb;
+          span.setAttribute(traceTypes.ATTR_RESPONSE_TTFB, ttfb);
+        }
+
         // Write the audio frame to the output stream
         await outputWriter.write(frame);
 
@@ -631,6 +677,10 @@ export function performTTSInference(
       }
       throw error;
     } finally {
+      if (!timedTextsFut.done) {
+        // Ensure downstream consumers don't hang on errors.
+        timedTextsFut.resolve(null);
+      }
       ttsStreamReader?.releaseLock();
       await ttsStream?.cancel();
       await outputWriter.close();
@@ -642,16 +692,11 @@ export function performTTSInference(
   const currentContext = otelContext.active();
 
   const inferenceTask = async (signal: AbortSignal) =>
-    tracer.startActiveSpan(async () => _performTTSInferenceImpl(signal), {
+    tracer.startActiveSpan(async (span) => _performTTSInferenceImpl(signal, span), {
       name: 'tts_node',
       context: currentContext,
     });
 
-  const genData: _TTSGenerationData = {
-    audioStream: audioOutputStream,
-    timedTextsFut,
-  };
-
   return [
     Task.from((controller) => inferenceTask(controller.signal), controller, 'performTTSInference'),
     genData,
@@ -719,7 +764,6 @@ export function performTextForwarding(
 
 export interface _AudioOut {
   audio: Array<AudioFrame>;
-  /** Future that will be set with the timestamp of the first frame's capture */
   firstFrameFut: Future<number>;
 }
 
@@ -807,7 +851,6 @@ export function performAudioForwarding(
   ];
 }
 
-// function_tool span is already implemented in tracableToolExecution below (line ~796)
 export function performToolExecutions({
   session,
   speechHandle,

package/src/voice/index.ts

@@ -5,6 +5,7 @@ export { Agent, AgentTask, StopResponse, type AgentOptions, type ModelSettings }
 export { AgentSession, type AgentSessionOptions, type VoiceOptions } from './agent_session.js';
 export * from './avatar/index.js';
 export * from './background_audio.js';
+export { type TextInputCallback, type TextInputEvent } from './client_events.js';
 export * from './events.js';
 export { type TimedString } from './io.js';
 export * from './report.js';

package/src/voice/report.test.ts

@@ -0,0 +1,117 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { ChatContext } from '../llm/chat_context.js';
+import type { VoiceOptions } from './agent_session.js';
+import { createSessionReport, sessionReportToJSON } from './report.js';
+
+function baseOptions(): VoiceOptions {
+  return {
+    maxToolSteps: 3,
+    preemptiveGeneration: false,
+    userAwayTimeout: 15,
+    useTtsAlignedTranscript: true,
+    turnHandling: {},
+  };
+}
+
+function serializeOptions(options: VoiceOptions) {
+  const report = createSessionReport({
+    jobId: 'job',
+    roomId: 'room-id',
+    room: 'room',
+    options,
+    events: [],
+    chatHistory: ChatContext.empty(),
+    enableRecording: false,
+    timestamp: 0,
+    startedAt: 0,
+  });
+
+  const payload = sessionReportToJSON(report);
+  return payload.options as Record<string, unknown>;
+}
+
+describe('sessionReportToJSON', () => {
+  it('serializes interruption and endpointing values from turnHandling', () => {
+    const options = baseOptions();
+    options.turnHandling = {
+      interruption: {
+        mode: 'adaptive',
+        discardAudioIfUninterruptible: false,
+        minDuration: 1200,
+        minWords: 2,
+      },
+      endpointing: {
+        minDelay: 900,
+        maxDelay: 4500,
+      },
+    };
+
+    const serialized = serializeOptions(options);
+    expect(serialized).toMatchObject({
+      allow_interruptions: true,
+      discard_audio_if_uninterruptible: false,
+      min_interruption_duration: 1200,
+      min_interruption_words: 2,
+      min_endpointing_delay: 900,
+      max_endpointing_delay: 4500,
+      max_tool_steps: 3,
+    });
+  });
+
+  it('prefers turnHandling values over deprecated flat fields', () => {
+    const options = baseOptions();
+    options.allowInterruptions = false;
+    options.discardAudioIfUninterruptible = true;
+    options.minInterruptionDuration = 400;
+    options.minInterruptionWords = 1;
+    options.minEndpointingDelay = 500;
+    options.maxEndpointingDelay = 2500;
+    options.turnHandling = {
+      interruption: {
+        mode: 'vad',
+        discardAudioIfUninterruptible: false,
+        minDuration: 1400,
+        minWords: 4,
+      },
+      endpointing: {
+        minDelay: 700,
+        maxDelay: 3900,
+      },
+    };
+
+    const serialized = serializeOptions(options);
+    expect(serialized).toMatchObject({
+      allow_interruptions: true,
+      discard_audio_if_uninterruptible: false,
+      min_interruption_duration: 1400,
+      min_interruption_words: 4,
+      min_endpointing_delay: 700,
+      max_endpointing_delay: 3900,
+      max_tool_steps: 3,
+    });
+  });
+
+  it('falls back to deprecated flat fields when turnHandling values are absent', () => {
+    const options = baseOptions();
+    options.allowInterruptions = false;
+    options.discardAudioIfUninterruptible = false;
+    options.minInterruptionDuration = 600;
+    options.minInterruptionWords = 3;
+    options.minEndpointingDelay = 1000;
+    options.maxEndpointingDelay = 5000;
+
+    const serialized = serializeOptions(options);
+    expect(serialized).toMatchObject({
+      allow_interruptions: false,
+      discard_audio_if_uninterruptible: false,
+      min_interruption_duration: 600,
+      min_interruption_words: 3,
+      min_endpointing_delay: 1000,
+      max_endpointing_delay: 5000,
+      max_tool_steps: 3,
+    });
+  });
+});

package/src/voice/report.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { ChatContext } from '../llm/chat_context.js';
+import { type ModelUsage, filterZeroValues } from '../metrics/model_usage.js';
 import type { VoiceOptions } from './agent_session.js';
 import type { AgentEvent } from './events.js';
 
@@ -23,6 +24,8 @@ export interface SessionReport {
   audioRecordingStartedAt?: number;
   /** Duration of the session in milliseconds */
   duration?: number;
+  /** Usage summaries for the session, one per model/provider combination */
+  modelUsage?: ModelUsage[];
 }
 
 export interface SessionReportOptions {
@@ -41,6 +44,8 @@ export interface SessionReportOptions {
   audioRecordingPath?: string;
   /** Timestamp when the audio recording started (milliseconds) */
   audioRecordingStartedAt?: number;
+  /** Usage summaries for the session, one per model/provider combination */
+  modelUsage?: ModelUsage[];
 }
 
 export function createSessionReport(opts: SessionReportOptions): SessionReport {
@@ -61,6 +66,7 @@ export function createSessionReport(opts: SessionReportOptions): SessionReport {
     audioRecordingStartedAt,
     duration:
       audioRecordingStartedAt !== undefined ? timestamp - audioRecordingStartedAt : undefined,
+    modelUsage: opts.modelUsage,
   };
 }
 
@@ -70,6 +76,22 @@ export function createSessionReport(opts: SessionReportOptions): SessionReport {
 //   - Uploads to LiveKit Cloud observability endpoint with JWT auth
 export function sessionReportToJSON(report: SessionReport): Record<string, unknown> {
   const events: Record<string, unknown>[] = [];
+  const interruptionConfig = report.options.turnHandling?.interruption;
+  const endpointingConfig = report.options.turnHandling?.endpointing;
+
+  // Keep backwards compatibility with deprecated fields
+  const allowInterruptions =
+    interruptionConfig?.mode !== undefined
+      ? interruptionConfig.mode !== false
+      : report.options.allowInterruptions;
+  const discardAudioIfUninterruptible =
+    interruptionConfig?.discardAudioIfUninterruptible ??
+    report.options.discardAudioIfUninterruptible;
+  const minInterruptionDuration =
+    interruptionConfig?.minDuration ?? report.options.minInterruptionDuration;
+  const minInterruptionWords = interruptionConfig?.minWords ?? report.options.minInterruptionWords;
+  const minEndpointingDelay = endpointingConfig?.minDelay ?? report.options.minEndpointingDelay;
+  const maxEndpointingDelay = endpointingConfig?.maxDelay ?? report.options.maxEndpointingDelay;
 
   for (const event of report.events) {
     if (event.type === 'metrics_collected') {
@@ -85,16 +107,17 @@ export function sessionReportToJSON(report: SessionReport): Record<string, unkno
     room: report.room,
     events,
     options: {
-      allow_interruptions: report.options.allowInterruptions,
-      discard_audio_if_uninterruptible: report.options.discardAudioIfUninterruptible,
-      min_interruption_duration: report.options.minInterruptionDuration,
-      min_interruption_words: report.options.minInterruptionWords,
-      min_endpointing_delay: report.options.minEndpointingDelay,
-      max_endpointing_delay: report.options.maxEndpointingDelay,
+      allow_interruptions: allowInterruptions,
+      discard_audio_if_uninterruptible: discardAudioIfUninterruptible,
+      min_interruption_duration: minInterruptionDuration,
+      min_interruption_words: minInterruptionWords,
+      min_endpointing_delay: minEndpointingDelay,
+      max_endpointing_delay: maxEndpointingDelay,
       max_tool_steps: report.options.maxToolSteps,
     },
     chat_history: report.chatHistory.toJSON({ excludeTimestamp: false }),
     enable_user_data_training: report.enableRecording,
     timestamp: report.timestamp,
+    usage: report.modelUsage ? report.modelUsage.map(filterZeroValues) : null,
   };
 }

package/src/voice/room_io/room_io.ts

@@ -12,17 +12,16 @@ import {
   type RemoteParticipant,
   type Room,
   RoomEvent,
-  type TextStreamInfo,
-  type TextStreamReader,
   TrackPublishOptions,
   TrackSource,
 } from '@livekit/rtc-node';
 import type { WritableStreamDefaultWriter } from 'node:stream/web';
-import { ATTRIBUTE_PUBLISH_ON_BEHALF, TOPIC_CHAT } from '../../constants.js';
+import { ATTRIBUTE_PUBLISH_ON_BEHALF } from '../../constants.js';
 import { log } from '../../log.js';
 import { IdentityTransform } from '../../stream/identity_transform.js';
-import { Future, Task } from '../../utils.js';
+import { Future, Task, waitForAbort } from '../../utils.js';
 import { type AgentSession } from '../agent_session.js';
+import type { TextInputCallback } from '../client_events.js';
 import {
   AgentSessionEventTypes,
   type AgentStateChangedEvent,
@@ -39,15 +38,7 @@ import {
   ParticipantTranscriptionOutput,
 } from './_output.js';
 
-export interface TextInputEvent {
-  text: string;
-  info: TextStreamInfo;
-  participant: RemoteParticipant;
-}
-
-export type TextInputCallback = (sess: AgentSession, ev: TextInputEvent) => void | Promise<void>;
-
-const DEFAULT_TEXT_INPUT_CALLBACK: TextInputCallback = (sess: AgentSession, ev: TextInputEvent) => {
+export const DEFAULT_TEXT_INPUT_CALLBACK: TextInputCallback = (sess, ev) => {
   sess.interrupt();
   sess.generateReply({ userInput: ev.text });
 };
@@ -146,8 +137,6 @@ export class RoomIO {
   private forwardUserTranscriptTask?: Task<void>;
   private initTask?: Task<void>;
 
-  private textStreamHandlerRegistered = false;
-
   private logger = log();
 
   constructor({
@@ -177,7 +166,10 @@ export class RoomIO {
       : this.inputOptions.participantIdentity ?? null;
   }
   private async init(signal: AbortSignal): Promise<void> {
-    await this.roomConnectedFuture.await;
+    await Promise.race([this.roomConnectedFuture.await, waitForAbort(signal)]);
+    if (signal.aborted) {
+      return;
+    }
 
     for (const participant of this.room.remoteParticipants.values()) {
       this.onParticipantConnected(participant);
@@ -186,7 +178,15 @@ export class RoomIO {
       return;
     }
 
-    const participant = await this.participantAvailableFuture.await;
+    const participant = await Promise.race([
+      this.participantAvailableFuture.await,
+      waitForAbort(signal),
+    ]);
+
+    if (!participant) {
+      return;
+    }
+
     this.setParticipant(participant.identity);
 
     // init agent outputs
@@ -271,37 +271,6 @@ export class RoomIO {
     }
   };
 
-  private onUserTextInput = (reader: TextStreamReader, participantInfo: { identity: string }) => {
-    if (participantInfo.identity !== this.participantIdentity) {
-      return;
-    }
-
-    const participant = this.room.remoteParticipants.get(participantInfo.identity);
-    if (!participant) {
-      this.logger.warn('participant not found, ignoring text input');
-      return;
-    }
-
-    const readText = async () => {
-      const text = await reader.readAll();
-
-      const textInputResult = this.inputOptions.textInputCallback!(this.agentSession, {
-        text,
-        info: reader.info,
-        participant,
-      });
-
-      // check if callback is a Promise
-      if (textInputResult instanceof Promise) {
-        await textInputResult;
-      }
-    };
-
-    readText().catch((error) => {
-      this.logger.error({ error }, 'Error reading text input');
-    });
-  };
-
   private async forwardUserTranscript(signal: AbortSignal): Promise<void> {
     const reader = this.userTranscriptStream.readable.getReader();
     try {
@@ -376,6 +345,10 @@ export class RoomIO {
     return this.participantAvailableFuture.done;
   }
 
+  get rtcRoom(): Room {
+    return this.room;
+  }
+
   get linkedParticipant(): RemoteParticipant | undefined {
     if (!this.isParticipantAvailable) {
       return undefined;
@@ -428,17 +401,6 @@ export class RoomIO {
   }
 
   start() {
-    if (this.inputOptions.textEnabled) {
-      try {
-        this.room.registerTextStreamHandler(TOPIC_CHAT, this.onUserTextInput);
-        this.textStreamHandlerRegistered = true;
-      } catch (error) {
-        if (this.inputOptions.textEnabled) {
-          this.logger.warn(`text stream handler for topic "${TOPIC_CHAT}" already set, ignoring`);
-        }
-      }
-    }
-
     // -- create inputs --
     if (this.inputOptions.audioEnabled) {
       this.audioInput = new ParticipantAudioInputStream({
@@ -514,11 +476,6 @@ export class RoomIO {
     this.agentSession.off(AgentSessionEventTypes.UserInputTranscribed, this.onUserInputTranscribed);
     this.agentSession.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
 
-    if (this.textStreamHandlerRegistered) {
-      this.room.unregisterTextStreamHandler(TOPIC_CHAT);
-      this.textStreamHandlerRegistered = false;
-    }
-
     await this.initTask?.cancelAndWait();
 
     // Close stream FIRST so reader.read() in forwardUserTranscript can exit.