@livekit/agents 1.0.47 → 1.1.0-dev.0

package/src/telemetry/traces.ts

@@ -22,8 +22,9 @@ import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
 import FormData from 'form-data';
 import { AccessToken } from 'livekit-server-sdk';
 import fs from 'node:fs/promises';
-import type { ChatContent, ChatItem } from '../llm/index.js';
+import type { ChatContent, ChatItem, ChatRole } from '../llm/index.js';
 import { enableOtelLogging } from '../log.js';
+import { filterZeroValues } from '../metrics/model_usage.js';
 import type { SessionReport } from '../voice/report.js';
 import { type SimpleLogRecord, SimpleOTLPHttpLogExporter } from './otel_http_exporter.js';
 import { flushPinoLogs, initPinoCloudExporter } from './pino_otel_transport.js';
@@ -285,24 +286,80 @@ export async function flushOtelLogs(): Promise<void> {
   await flushPinoLogs();
 }
 
+/** Proto-compatible role enum values. */
+type ProtoRole = 'DEVELOPER' | 'SYSTEM' | 'USER' | 'ASSISTANT';
+
+const ROLE_MAP: Record<ChatRole, ProtoRole> = {
+  developer: 'DEVELOPER',
+  system: 'SYSTEM',
+  user: 'USER',
+  assistant: 'ASSISTANT',
+};
+
+interface ProtoMetricsReport {
+  startedSpeakingAt?: string;
+  stoppedSpeakingAt?: string;
+  transcriptionDelay?: number;
+  endOfTurnDelay?: number;
+  onUserTurnCompletedDelay?: number;
+  llmNodeTtft?: number;
+  ttsNodeTtfb?: number;
+  e2eLatency?: number;
+}
+
+interface ProtoMessage {
+  id: string;
+  role: ProtoRole;
+  content: { text: ChatContent }[];
+  createdAt: string;
+  interrupted?: boolean;
+  extra?: Record<string, unknown>;
+  transcriptConfidence?: number;
+  metrics?: ProtoMetricsReport;
+}
+
+interface ProtoFunctionCall {
+  id: string;
+  callId: string;
+  arguments: string | Record<string, unknown>;
+  name: string;
+  createdAt: string;
+}
+
+interface ProtoFunctionCallOutput {
+  id: string;
+  name: string;
+  callId: string;
+  output: string;
+  isError: boolean;
+  createdAt: string;
+}
+
+interface ProtoAgentHandoff {
+  id: string;
+  newAgentId: string;
+  createdAt: string;
+  oldAgentId?: string;
+}
+
+interface ProtoChatItem {
+  message?: ProtoMessage;
+  functionCall?: ProtoFunctionCall;
+  functionCallOutput?: ProtoFunctionCallOutput;
+  agentHandoff?: ProtoAgentHandoff;
+}
+
 /**
  * Convert ChatItem to proto-compatible dictionary format.
  * TODO: Use actual agent_session proto types once @livekit/protocol v1.43.1+ is published
  */
-function chatItemToProto(item: ChatItem): Record<string, any> {
-  const itemDict: Record<string, any> = {};
+function chatItemToProto(item: ChatItem): ProtoChatItem {
+  const itemDict: ProtoChatItem = {};
 
   if (item.type === 'message') {
-    const roleMap: Record<string, string> = {
-      developer: 'DEVELOPER',
-      system: 'SYSTEM',
-      user: 'USER',
-      assistant: 'ASSISTANT',
-    };
-
-    const msg: Record<string, any> = {
+    const msg: ProtoMessage = {
       id: item.id,
-      role: roleMap[item.role] || item.role.toUpperCase(),
+      role: ROLE_MAP[item.role] ?? (item.role.toUpperCase() as ProtoRole),
       content: item.content.map((c: ChatContent) => ({ text: c })),
       createdAt: toRFC3339(item.createdAt),
     };
@@ -311,44 +368,43 @@ function chatItemToProto(item: ChatItem): Record<string, any> {
       msg.interrupted = item.interrupted;
     }
 
-    // TODO(brian): Add extra and transcriptConfidence to ChatMessage
-    // if (item.extra && Object.keys(item.extra).length > 0) {
-    //   msg.extra = item.extra;
-    // }
-
-    // if (item.transcriptConfidence !== undefined && item.transcriptConfidence !== null) {
-    //   msg.transcriptConfidence = item.transcriptConfidence;
-    // }
-
-    // TODO(brian): Add metrics to ChatMessage
-    // const metrics = item.metrics || {};
-    // if (Object.keys(metrics).length > 0) {
-    //   msg.metrics = {};
-    //   if (metrics.started_speaking_at) {
-    //     msg.metrics.startedSpeakingAt = toRFC3339(metrics.started_speaking_at);
-    //   }
-    //   if (metrics.stopped_speaking_at) {
-    //     msg.metrics.stoppedSpeakingAt = toRFC3339(metrics.stopped_speaking_at);
-    //   }
-    //   if (metrics.transcription_delay !== undefined) {
-    //     msg.metrics.transcriptionDelay = metrics.transcription_delay;
-    //   }
-    //   if (metrics.end_of_turn_delay !== undefined) {
-    //     msg.metrics.endOfTurnDelay = metrics.end_of_turn_delay;
-    //   }
-    //   if (metrics.on_user_turn_completed_delay !== undefined) {
-    //     msg.metrics.onUserTurnCompletedDelay = metrics.on_user_turn_completed_delay;
-    //   }
-    //   if (metrics.llm_node_ttft !== undefined) {
-    //     msg.metrics.llmNodeTtft = metrics.llm_node_ttft;
-    //   }
-    //   if (metrics.tts_node_ttfb !== undefined) {
-    //     msg.metrics.ttsNodeTtfb = metrics.tts_node_ttfb;
-    //   }
-    //   if (metrics.e2e_latency !== undefined) {
-    //     msg.metrics.e2eLatency = metrics.e2e_latency;
-    //   }
-    // }
+    if (item.extra && Object.keys(item.extra).length > 0) {
+      msg.extra = item.extra;
+    }
+
+    if (item.transcriptConfidence !== undefined) {
+      msg.transcriptConfidence = item.transcriptConfidence;
+    }
+
+    const metrics = item.metrics;
+    if (metrics && Object.keys(metrics).length > 0) {
+      const protoMetrics: ProtoMetricsReport = {};
+      if (metrics.startedSpeakingAt !== undefined) {
+        protoMetrics.startedSpeakingAt = toRFC3339(metrics.startedSpeakingAt * 1000);
+      }
+      if (metrics.stoppedSpeakingAt !== undefined) {
+        protoMetrics.stoppedSpeakingAt = toRFC3339(metrics.stoppedSpeakingAt * 1000);
+      }
+      if (metrics.transcriptionDelay !== undefined) {
+        protoMetrics.transcriptionDelay = metrics.transcriptionDelay;
+      }
+      if (metrics.endOfTurnDelay !== undefined) {
+        protoMetrics.endOfTurnDelay = metrics.endOfTurnDelay;
+      }
+      if (metrics.onUserTurnCompletedDelay !== undefined) {
+        protoMetrics.onUserTurnCompletedDelay = metrics.onUserTurnCompletedDelay;
+      }
+      if (metrics.llmNodeTtft !== undefined) {
+        protoMetrics.llmNodeTtft = metrics.llmNodeTtft;
+      }
+      if (metrics.ttsNodeTtfb !== undefined) {
+        protoMetrics.ttsNodeTtfb = metrics.ttsNodeTtfb;
+      }
+      if (metrics.e2eLatency !== undefined) {
+        protoMetrics.e2eLatency = metrics.e2eLatency;
+      }
+      msg.metrics = protoMetrics;
+    }
 
     itemDict.message = msg;
   } else if (item.type === 'function_call') {
@@ -369,7 +425,7 @@ function chatItemToProto(item: ChatItem): Record<string, any> {
       createdAt: toRFC3339(item.createdAt),
     };
   } else if (item.type === 'agent_handoff') {
-    const handoff: Record<string, any> = {
+    const handoff: ProtoAgentHandoff = {
       id: item.id,
       newAgentId: item.newAgentId,
       createdAt: toRFC3339(item.createdAt),
@@ -397,9 +453,7 @@ function chatItemToProto(item: ChatItem): Record<string, any> {
 }
 
 /**
- * Convert timestamp to RFC3339 format matching Python's _to_rfc3339.
- * Note: TypeScript createdAt is in milliseconds (Date.now()), not seconds like Python.
- * @internal
+ * Convert timestamp to RFC3339 format
  */
 function toRFC3339(valueMs: number | Date): string {
   // valueMs is already in milliseconds (from Date.now())
@@ -445,6 +499,8 @@ export async function uploadSessionReport(options: {
     'logger.name': 'chat_history',
   };
 
+  const usage = report.modelUsage?.map(filterZeroValues) || null;
+
   logRecords.push({
     body: 'session report',
     timestampMs: report.startedAt || report.timestamp || 0,
@@ -453,6 +509,7 @@ export async function uploadSessionReport(options: {
       'session.options': report.options || {},
       'session.report_timestamp': report.timestamp,
       agent_name: agentName,
+      usage,
     },
   });
 

package/src/tts/tts.ts

@@ -96,6 +96,30 @@ export abstract class TTS extends (EventEmitter as new () => TypedEmitter<TTSCal
     return this.#numChannels;
   }
 
+  /**
+   * Get the model name/identifier for this TTS instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model(): string {
+    return 'unknown';
+  }
+
+  /**
+   * Get the provider name for this TTS instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider(): string {
+    return 'unknown';
+  }
+
   /**
    * Receives text and returns synthesis in the form of a {@link ChunkedStream}
    */
@@ -159,6 +183,8 @@ export abstract class SynthesizeStream
   #metricsText = '';
   #monitorMetricsTask?: Promise<void>;
   #ttsRequestSpan?: Span;
+  #inputTokens = 0;
+  #outputTokens = 0;
 
   constructor(tts: TTS, connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS) {
     this.#tts = tts;
@@ -284,6 +310,18 @@ export abstract class SynthesizeStream
     }
   }
 
+  /**
+   * Set token usage for token-based TTS billing (e.g., OpenAI TTS).
+   * Plugins should call this method to report token usage.
+   */
+  protected setTokenUsage({
+    inputTokens = 0,
+    outputTokens = 0,
+  }: { inputTokens?: number; outputTokens?: number } = {}): void {
+    this.#inputTokens = inputTokens;
+    this.#outputTokens = outputTokens;
+  }
+
   protected async monitorMetrics() {
     const startTime = process.hrtime.bigint();
     let audioDurationMs = 0;
@@ -305,12 +343,22 @@ export abstract class SynthesizeStream
           audioDurationMs: roundedAudioDurationMs,
           cancelled: this.abortController.signal.aborted,
           label: this.#tts.label,
-          streamed: false,
+          inputTokens: this.#inputTokens,
+          outputTokens: this.#outputTokens,
+          streamed: true,
+          metadata: {
+            modelProvider: this.#tts.provider,
+            modelName: this.#tts.model,
+          },
         };
         if (this.#ttsRequestSpan) {
           this.#ttsRequestSpan.setAttribute(traceTypes.ATTR_TTS_METRICS, JSON.stringify(metrics));
         }
         this.#tts.emit('metrics_collected', metrics);
+
+        // Reset token usage after emitting metrics for the next segment
+        this.#inputTokens = 0;
+        this.#outputTokens = 0;
       }
     };
 
@@ -434,6 +482,8 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
   #ttsRequestSpan?: Span;
   private _connOptions: APIConnectOptions;
   private logger = log();
+  #inputTokens = 0;
+  #outputTokens = 0;
 
   protected abortController = new AbortController();
 
@@ -541,6 +591,18 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
     return this.abortController.signal;
   }
 
+  /**
+   * Set token usage for token-based TTS billing (e.g., OpenAI TTS).
+   * Plugins should call this method to report token usage.
+   */
+  protected setTokenUsage({
+    inputTokens = 0,
+    outputTokens = 0,
+  }: { inputTokens?: number; outputTokens?: number } = {}): void {
+    this.#inputTokens = inputTokens;
+    this.#outputTokens = outputTokens;
+  }
+
   protected async monitorMetrics() {
     const startTime = process.hrtime.bigint();
     let audioDurationMs = 0;
@@ -568,7 +630,13 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
       audioDurationMs: Math.round(audioDurationMs),
       cancelled: false, // TODO(AJS-186): support ChunkedStream with 1.0 - add this.abortController.signal.aborted here
       label: this.#tts.label,
+      inputTokens: this.#inputTokens,
+      outputTokens: this.#outputTokens,
       streamed: false,
+      metadata: {
+        modelProvider: this.#tts.provider,
+        modelName: this.#tts.model,
+      },
     };
 
     if (this.#ttsRequestSpan) {

package/src/utils.ts

@@ -173,6 +173,11 @@ export class Future<T = void> {
     this.#rejected = true;
     this.#error = error;
     this.#rejectPromise(error);
+    // Python calls Future.exception() right after set_exception() to silence
+    // "exception was never retrieved" warnings. In JS, consume the rejection
+    // immediately so Node does not emit unhandled-rejection noise before a
+    // later await/catch observes it.
+    void this.#await.catch(() => undefined);
   }
 }
 

package/src/voice/agent.ts

@@ -35,6 +35,9 @@ import { type AgentActivity, agentActivityStorage } from './agent_activity.js';
 import type { AgentSession, TurnDetectionMode } from './agent_session.js';
 import type { TimedString } from './io.js';
 import type { SpeechHandle } from './speech_handle.js';
+import type { InterruptionOptions } from './turn_config/interruption.js';
+import type { TurnHandlingOptions } from './turn_config/turn_handling.js';
+import { migrateLegacyOptions } from './turn_config/utils.js';
 
 export const functionCallStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();
 export const speechHandleStorage = new AsyncLocalStorage<SpeechHandle>();
@@ -110,6 +113,7 @@ export interface AgentOptions<UserData> {
   instructions: string;
   chatCtx?: ChatContext;
   tools?: ToolContext<UserData>;
+  /** @deprecated use turnHandling instead */
   turnDetection?: TurnDetectionMode;
   stt?: STT | STTModelString;
   vad?: VAD;
@@ -117,16 +121,19 @@ export interface AgentOptions<UserData> {
   tts?: TTS | TTSModelString;
   allowInterruptions?: boolean;
   minConsecutiveSpeechDelay?: number;
+  turnHandling?: TurnHandlingOptions;
   useTtsAlignedTranscript?: boolean;
 }
 
 export class Agent<UserData = any> {
   private _id: string;
-  private turnDetection?: TurnDetectionMode;
   private _stt?: STT;
   private _vad?: VAD;
   private _llm?: LLM | RealtimeModel;
   private _tts?: TTS;
+  private turnHandling?: TurnHandlingOptions;
+  private _interruptionDetection: InterruptionOptions['mode'];
+  private _allowInterruptions?: boolean;
   private _useTtsAlignedTranscript?: boolean;
 
   /** @internal */
@@ -151,7 +158,9 @@ export class Agent<UserData = any> {
     vad,
     llm,
     tts,
+    turnHandling,
     useTtsAlignedTranscript,
+    allowInterruptions,
   }: AgentOptions<UserData>) {
     if (id) {
       this._id = id;
@@ -176,7 +185,12 @@ export class Agent<UserData = any> {
         })
       : ChatContext.empty();
 
-    this.turnDetection = turnDetection;
+    const migratedOptions = migrateLegacyOptions({
+      turnDetection,
+      options: { turnHandling, allowInterruptions },
+    });
+    this.turnHandling = migratedOptions.options.turnHandling;
+
     this._vad = vad;
 
     if (typeof stt === 'string') {
@@ -197,6 +211,10 @@ export class Agent<UserData = any> {
       this._tts = tts;
     }
 
+    this._interruptionDetection = this.turnHandling?.interruption.mode;
+    if (this.turnHandling?.interruption.mode !== undefined) {
+      this._allowInterruptions = !!this.turnHandling.interruption.mode;
+    }
     this._useTtsAlignedTranscript = useTtsAlignedTranscript;
 
     this._agentActivity = undefined;
@@ -242,6 +260,14 @@ export class Agent<UserData = any> {
     return this.getActivityOrThrow().agentSession as AgentSession<UserData>;
   }
 
+  get interruptionDetection(): InterruptionOptions['mode'] {
+    return this._interruptionDetection;
+  }
+
+  get allowInterruptions(): boolean | undefined {
+    return this._allowInterruptions;
+  }
+
   async onEnter(): Promise<void> {}
 
   async onExit(): Promise<void> {}
@@ -302,6 +328,17 @@ export class Agent<UserData = any> {
     this._agentActivity.updateChatCtx(chatCtx);
   }
 
+  // TODO(parity): Add when AgentConfigUpdate is ported to ChatContext.
+  async updateTools(tools: ToolContext): Promise<void> {
+    if (!this._agentActivity) {
+      this._tools = { ...tools };
+      this._chatCtx = this._chatCtx.copy({ toolCtx: this._tools });
+      return;
+    }
+
+    await this._agentActivity.updateTools(tools);
+  }
+
   static default = {
     async sttNode(
       agent: Agent,
@@ -330,7 +367,8 @@ export class Agent<UserData = any> {
 
       // Set startTimeOffset to provide linear timestamps across reconnections
       const audioInputStartedAt =
-        activity.agentSession._recorderIO?.recordingStartedAt ?? // Use recording start time if available
+        activity.inputStartedAt ?? // Use input started at proxied from AudioRecognition if available
+        activity.agentSession._recorderIO?.recordingStartedAt ?? // Fallback to recording start time if available
         activity.agentSession._startedAt ?? // Fallback to session start time
         Date.now(); // Fallback to current time