@livekit/agents 1.0.47 → 1.1.0-dev.0

package/src/llm/chat_context.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame, VideoFrame } from '@livekit/rtc-node';
 import { createImmutableArray, shortuuid } from '../utils.js';
+import type { LLM } from './llm.js';
 import { type ProviderFormat, toChatCtx } from './provider_format/index.js';
 import type { JSONObject, JSONValue, ToolContext } from './tool_context.js';
 
@@ -80,6 +81,17 @@ export function createAudioContent(params: {
   };
 }
 
+export interface MetricsReport {
+  startedSpeakingAt?: number;
+  stoppedSpeakingAt?: number;
+  transcriptionDelay?: number;
+  endOfTurnDelay?: number;
+  onUserTurnCompletedDelay?: number;
+  llmNodeTtft?: number;
+  ttsNodeTtfb?: number;
+  e2eLatency?: number;
+}
+
 export class ChatMessage {
   readonly id: string;
 
@@ -91,6 +103,12 @@ export class ChatMessage {
 
   interrupted: boolean;
 
+  transcriptConfidence?: number;
+
+  extra: Record<string, unknown>;
+
+  metrics: MetricsReport;
+
   hash?: Uint8Array;
 
   createdAt: number;
@@ -101,6 +119,9 @@ export class ChatMessage {
     id?: string;
     interrupted?: boolean;
     createdAt?: number;
+    transcriptConfidence?: number;
+    metrics?: MetricsReport;
+    extra?: Record<string, unknown>;
   }) {
     const {
       role,
@@ -108,12 +129,18 @@ export class ChatMessage {
       id = shortuuid('item_'),
       interrupted = false,
       createdAt = Date.now(),
+      transcriptConfidence,
+      metrics = {},
+      extra = {},
     } = params;
     this.id = id;
     this.role = role;
     this.content = Array.isArray(content) ? content : [content];
     this.interrupted = interrupted;
     this.createdAt = createdAt;
+    this.transcriptConfidence = transcriptConfidence;
+    this.metrics = metrics;
+    this.extra = extra;
   }
 
   static create(params: {
@@ -122,6 +149,9 @@ export class ChatMessage {
     id?: string;
     interrupted?: boolean;
     createdAt?: number;
+    transcriptConfidence?: number;
+    metrics?: MetricsReport;
+    extra?: Record<string, unknown>;
   }) {
     return new ChatMessage(params);
   }
@@ -172,6 +202,16 @@ export class ChatMessage {
       result.createdAt = this.createdAt;
     }
 
+    if (this.transcriptConfidence !== undefined) {
+      result.transcriptConfidence = this.transcriptConfidence;
+    }
+    if (Object.keys(this.metrics).length > 0) {
+      result.metrics = { ...this.metrics };
+    }
+    if (Object.keys(this.extra).length > 0) {
+      result.extra = this.extra as JSONValue;
+    }
+
     return result;
   }
 }
@@ -401,6 +441,7 @@ export class AgentHandoffItem {
   }
 }
 
+// TODO(parity): Add AgentConfigUpdate type to ChatItem union
 export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput | AgentHandoffItem;
 
 export class ChatContext {
@@ -431,6 +472,9 @@ export class ChatContext {
     id?: string;
     interrupted?: boolean;
     createdAt?: number;
+    transcriptConfidence?: number;
+    metrics?: MetricsReport;
+    extra?: Record<string, unknown>;
   }): ChatMessage {
     const msg = new ChatMessage(params);
     if (params.createdAt !== undefined) {
@@ -463,11 +507,13 @@ export class ChatContext {
     return idx !== -1 ? idx : undefined;
   }
 
+  // TODO(parity): Add excludeConfigUpdate option when AgentConfigUpdate is ported
   copy(
     options: {
       excludeFunctionCall?: boolean;
       excludeInstructions?: boolean;
       excludeEmptyMessage?: boolean;
+      excludeHandoff?: boolean;
       toolCtx?: ToolContext<any>; // eslint-disable-line @typescript-eslint/no-explicit-any
     } = {},
   ): ChatContext {
@@ -475,6 +521,7 @@ export class ChatContext {
       excludeFunctionCall = false,
       excludeInstructions = false,
       excludeEmptyMessage = false,
+      excludeHandoff = false,
       toolCtx,
     } = options;
     const items: ChatItem[] = [];
@@ -500,6 +547,10 @@ export class ChatContext {
         continue;
       }
 
+      if (excludeHandoff && item.type === 'agent_handoff') {
+        continue;
+      }
+
       if (toolCtx !== undefined && isToolCallOrOutput(item) && toolCtx[item.name] === undefined) {
         continue;
       }
@@ -510,6 +561,7 @@ export class ChatContext {
     return new ChatContext(items);
   }
 
+  // TODO(parity): Add excludeConfigUpdate option when AgentConfigUpdate is ported
   merge(
     other: ChatContext,
     options: {
@@ -606,6 +658,9 @@ export class ChatContext {
           id: item.id,
           interrupted: item.interrupted,
           createdAt: item.createdAt,
+          transcriptConfidence: item.transcriptConfidence,
+          metrics: item.metrics,
+          extra: item.extra,
         });
 
         // Filter content based on options
@@ -762,6 +817,112 @@ export class ChatContext {
     return true;
   }
 
+  async _summarize(llm: LLM, options: { keepLastTurns?: number } = {}): Promise<ChatContext> {
+    const { keepLastTurns = 2 } = options;
+
+    const toSummarize: ChatMessage[] = [];
+    for (const item of this._items) {
+      if (item.type !== 'message') continue;
+      if (item.role !== 'user' && item.role !== 'assistant') continue;
+      if (item.extra?.is_summary === true) continue;
+
+      const text = (item.textContent ?? '').trim();
+      if (text) {
+        toSummarize.push(item);
+      }
+    }
+
+    if (toSummarize.length === 0) {
+      return this;
+    }
+
+    const tailN = Math.max(0, Math.min(toSummarize.length, keepLastTurns * 2));
+    let head: ChatMessage[];
+    let tail: ChatMessage[];
+    if (tailN === 0) {
+      head = toSummarize;
+      tail = [];
+    } else {
+      head = toSummarize.slice(0, -tailN);
+      tail = toSummarize.slice(-tailN);
+    }
+
+    if (head.length === 0) {
+      return this;
+    }
+
+    const sourceText = head
+      .map((m) => `${m.role}: ${(m.textContent ?? '').trim()}`)
+      .join('\n')
+      .trim();
+
+    if (!sourceText) {
+      return this;
+    }
+
+    // TODO: refactor this into LLMStream.collect API.
+    const promptCtx = new ChatContext();
+    promptCtx.addMessage({
+      role: 'system',
+      content:
+        'Compress older chat history into a short, faithful summary.\n' +
+        'Focus on user goals, constraints, decisions, key facts/preferences/entities, and pending tasks.\n' +
+        'Exclude chit-chat and greetings. Be concise.',
+    });
+    promptCtx.addMessage({
+      role: 'user',
+      content: `Conversation to summarize:\n\n${sourceText}`,
+    });
+
+    const chunks: string[] = [];
+    for await (const chunk of llm.chat({ chatCtx: promptCtx })) {
+      if (chunk.delta?.content) {
+        chunks.push(chunk.delta.content);
+      }
+    }
+
+    const summary = chunks.join('').trim();
+    if (!summary) {
+      return this;
+    }
+
+    const tailStartTs = tail.length > 0 ? tail[0]!.createdAt : Infinity;
+
+    const preserved: ChatItem[] = [];
+    for (const it of this._items) {
+      if (
+        (it.type === 'function_call' || it.type === 'function_call_output') &&
+        it.createdAt < tailStartTs
+      ) {
+        continue;
+      }
+
+      if (it.type === 'message' && (it.role === 'user' || it.role === 'assistant')) {
+        continue;
+      }
+
+      preserved.push(it);
+    }
+
+    this._items = preserved;
+
+    const createdAtHint =
+      tail.length > 0 ? tail[0]!.createdAt - 1e-3 : head[head.length - 1]!.createdAt + 1e-3;
+
+    this.addMessage({
+      role: 'assistant',
+      content: `[history summary]\n${summary}`,
+      createdAt: createdAtHint,
+      extra: { is_summary: true },
+    });
+
+    for (const msg of tail) {
+      this.insert(msg);
+    }
+
+    return this;
+  }
+
   /**
    * Indicates whether the context is read-only
    */

package/src/llm/index.ts

@@ -6,6 +6,7 @@ export {
   isFunctionTool,
   tool,
   ToolError,
+  ToolFlag,
   type AgentHandoff,
   type FunctionTool,
   type ProviderDefinedTool,
@@ -29,6 +30,7 @@ export {
   type ChatItem,
   type ChatRole,
   type ImageContent,
+  type MetricsReport,
 } from './chat_context.js';
 
 export type { ProviderFormat } from './provider_format/index.js';

package/src/llm/llm.ts

@@ -65,6 +65,18 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCal
     return 'unknown';
   }
 
+  /**
+   * Get the provider name for this LLM instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider(): string {
+    return 'unknown';
+  }
+
   /**
    * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
    */
@@ -248,6 +260,10 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
         }
         return (usage?.completionTokens || 0) / (durationMs / 1000);
       })(),
+      metadata: {
+        modelProvider: this.#llm.provider,
+        modelName: this.#llm.model,
+      },
     };
 
     if (this.#llmRequestSpan) {

package/src/llm/realtime.ts

@@ -73,6 +73,10 @@ export abstract class RealtimeModel {
   /** The model name/identifier used by this realtime model */
   abstract get model(): string;
 
+  get provider(): string {
+    return 'unknown';
+  }
+
   abstract session(): RealtimeSession;
 
   abstract close(): Promise<void>;

package/src/llm/tool_context.ts

@@ -80,6 +80,13 @@ export class ToolError extends Error {
   }
 }
 
+export const ToolFlag = {
+  NONE: 0,
+  IGNORE_ON_ENTER: 1 << 0,
+} as const;
+
+export type ToolFlag = (typeof ToolFlag)[keyof typeof ToolFlag];
+
 export interface AgentHandoff {
   /**
    * The agent to handoff to.
@@ -178,6 +185,8 @@ export interface FunctionTool<
    */
   execute: ToolExecuteFunction<Parameters, UserData, Result>;
 
+  flags: number;
+
   [FUNCTION_TOOL_SYMBOL]: true;
 }
 
@@ -242,10 +251,12 @@ export function tool<
   description,
   parameters,
   execute,
+  flags,
 }: {
   description: string;
   parameters: Schema;
   execute: ToolExecuteFunction<InferToolInput<Schema>, UserData, Result>;
+  flags?: number;
 }): FunctionTool<InferToolInput<Schema>, UserData, Result>;
 
 /**
@@ -254,10 +265,12 @@ export function tool<
 export function tool<UserData = UnknownUserData, Result = unknown>({
   description,
   execute,
+  flags,
 }: {
   description: string;
   parameters?: never;
   execute: ToolExecuteFunction<Record<string, never>, UserData, Result>;
+  flags?: number;
 }): FunctionTool<Record<string, never>, UserData, Result>;
 
 /**
@@ -295,6 +308,7 @@ export function tool(tool: any): any {
       description: tool.description,
       parameters,
       execute: tool.execute,
+      flags: tool.flags ?? ToolFlag.NONE,
       [TOOL_SYMBOL]: true,
       [FUNCTION_TOOL_SYMBOL]: true,
     };

package/src/metrics/base.ts

@@ -2,13 +2,21 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+export type MetricsMetadata = {
+  /** The provider name (e.g., 'openai', 'anthropic'). */
+  modelProvider?: string;
+  /** The model name (e.g., 'gpt-4o', 'claude-3-5-sonnet'). */
+  modelName?: string;
+};
+
 export type AgentMetrics =
   | STTMetrics
   | LLMMetrics
   | TTSMetrics
   | VADMetrics
   | EOUMetrics
-  | RealtimeModelMetrics;
+  | RealtimeModelMetrics
+  | InterruptionMetrics;
 
 export type LLMMetrics = {
   type: 'llm_metrics';
@@ -26,6 +34,8 @@ export type LLMMetrics = {
   totalTokens: number;
   tokensPerSecond: number;
   speechId?: string;
+  /** Metadata for model provider and name tracking. */
+  metadata?: MetricsMetadata;
 };
 
 export type STTMetrics = {
@@ -41,10 +51,16 @@ export type STTMetrics = {
    * The duration of the pushed audio in milliseconds.
    */
   audioDurationMs: number;
+  /** Input audio tokens (for token-based billing). */
+  inputTokens?: number;
+  /** Output text tokens (for token-based billing). */
+  outputTokens?: number;
   /**
    * Whether the STT is streaming (e.g using websocket).
    */
   streamed: boolean;
+  /** Metadata for model provider and name tracking. */
+  metadata?: MetricsMetadata;
 };
 
 export type TTSMetrics = {
@@ -59,10 +75,17 @@ export type TTSMetrics = {
   /** Generated audio duration in milliseconds. */
   audioDurationMs: number;
   cancelled: boolean;
+  /** Number of characters synthesized (for character-based billing). */
   charactersCount: number;
+  /** Input text tokens (for token-based billing, e.g., OpenAI TTS). */
+  inputTokens?: number;
+  /** Output audio tokens (for token-based billing, e.g., OpenAI TTS). */
+  outputTokens?: number;
   streamed: boolean;
   segmentId?: string;
   speechId?: string;
+  /** Metadata for model provider and name tracking. */
+  metadata?: MetricsMetadata;
 };
 
 export type VADMetrics = {
@@ -133,6 +156,10 @@ export type RealtimeModelMetrics = {
    * The duration of the response from created to done in milliseconds.
    */
   durationMs: number;
+  /**
+   * The duration of the session connection in milliseconds (for session-based billing like xAI).
+   */
+  sessionDurationMs?: number;
   /**
    * Time to first audio token in milliseconds. -1 if no audio token was sent.
    */
@@ -165,4 +192,24 @@ export type RealtimeModelMetrics = {
    * Details about the output tokens used in the Response.
    */
   outputTokenDetails: RealtimeModelMetricsOutputTokenDetails;
+  /** Metadata for model provider and name tracking. */
+  metadata?: MetricsMetadata;
+};
+
+export type InterruptionMetrics = {
+  type: 'interruption_metrics';
+  timestamp: number;
+  /** Latest RTT time taken to perform inference, in milliseconds. */
+  totalDuration: number;
+  /** Latest time taken by the model side, in milliseconds. */
+  predictionDuration: number;
+  /** Latest total time from onset of speech to final prediction, in milliseconds. */
+  detectionDelay: number;
+  /** Number of interruptions detected (incremental). */
+  numInterruptions: number;
+  /** Number of backchannels detected (incremental). */
+  numBackchannels: number;
+  /** Number of requests sent to the model (incremental). */
+  numRequests: number;
+  metadata?: MetricsMetadata;
 };

package/src/metrics/index.ts

@@ -5,11 +5,22 @@
 export type {
   AgentMetrics,
   EOUMetrics,
+  InterruptionMetrics,
   LLMMetrics,
+  MetricsMetadata,
   RealtimeModelMetrics,
   STTMetrics,
   TTSMetrics,
   VADMetrics,
 } from './base.js';
+export {
+  filterZeroValues,
+  ModelUsageCollector,
+  type InterruptionModelUsage,
+  type LLMModelUsage,
+  type ModelUsage,
+  type STTModelUsage,
+  type TTSModelUsage,
+} from './model_usage.js';
 export { UsageCollector, type UsageSummary } from './usage_collector.js';
 export { logMetrics } from './utils.js';