@livekit/agents 1.1.0 → 1.2.1

package/src/llm/chat_context.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame, VideoFrame } from '@livekit/rtc-node';
 import { createImmutableArray, shortuuid } from '../utils.js';
+import type { LLM } from './llm.js';
 import { type ProviderFormat, toChatCtx } from './provider_format/index.js';
 import type { JSONObject, JSONValue, ToolContext } from './tool_context.js';
 
@@ -80,6 +81,17 @@ export function createAudioContent(params: {
   };
 }
 
+export interface MetricsReport {
+  startedSpeakingAt?: number;
+  stoppedSpeakingAt?: number;
+  transcriptionDelay?: number;
+  endOfTurnDelay?: number;
+  onUserTurnCompletedDelay?: number;
+  llmNodeTtft?: number;
+  ttsNodeTtfb?: number;
+  e2eLatency?: number;
+}
+
 export class ChatMessage {
   readonly id: string;
 
@@ -91,6 +103,12 @@ export class ChatMessage {
 
   interrupted: boolean;
 
+  transcriptConfidence?: number;
+
+  extra: Record<string, unknown>;
+
+  metrics: MetricsReport;
+
   hash?: Uint8Array;
 
   createdAt: number;
@@ -101,6 +119,9 @@ export class ChatMessage {
     id?: string;
     interrupted?: boolean;
     createdAt?: number;
+    transcriptConfidence?: number;
+    metrics?: MetricsReport;
+    extra?: Record<string, unknown>;
   }) {
     const {
       role,
@@ -108,12 +129,18 @@ export class ChatMessage {
       id = shortuuid('item_'),
       interrupted = false,
       createdAt = Date.now(),
+      transcriptConfidence,
+      metrics = {},
+      extra = {},
     } = params;
     this.id = id;
     this.role = role;
     this.content = Array.isArray(content) ? content : [content];
     this.interrupted = interrupted;
     this.createdAt = createdAt;
+    this.transcriptConfidence = transcriptConfidence;
+    this.metrics = metrics;
+    this.extra = extra;
   }
 
   static create(params: {
@@ -122,6 +149,9 @@ export class ChatMessage {
     id?: string;
     interrupted?: boolean;
     createdAt?: number;
+    transcriptConfidence?: number;
+    metrics?: MetricsReport;
+    extra?: Record<string, unknown>;
   }) {
     return new ChatMessage(params);
   }
@@ -172,6 +202,16 @@ export class ChatMessage {
       result.createdAt = this.createdAt;
     }
 
+    if (this.transcriptConfidence !== undefined) {
+      result.transcriptConfidence = this.transcriptConfidence;
+    }
+    if (Object.keys(this.metrics).length > 0) {
+      result.metrics = { ...this.metrics };
+    }
+    if (Object.keys(this.extra).length > 0) {
+      result.extra = this.extra as JSONValue;
+    }
+
     return result;
   }
 }
@@ -189,19 +229,53 @@ export class FunctionCall {
 
   createdAt: number;
 
+  extra: Record<string, unknown>;
+  /**
+   * Optional grouping identifier for parallel tool calls.
+   */
+  groupId?: string;
+
+  /**
+   * Opaque signature for Gemini thinking mode.
+   * When using Gemini 3+ models with thinking enabled, this signature must be
+   * preserved and returned with function responses to maintain thought context.
+   */
+  thoughtSignature?: string;
+
   constructor(params: {
     callId: string;
     name: string;
     args: string;
     id?: string;
     createdAt?: number;
+    extra?: Record<string, unknown>;
+    groupId?: string;
+    thoughtSignature?: string;
   }) {
-    const { callId, name, args, id = shortuuid('item_'), createdAt = Date.now() } = params;
+    const {
+      callId,
+      name,
+      args,
+      id = shortuuid('item_'),
+      createdAt = Date.now(),
+      extra = {},
+      groupId,
+      thoughtSignature,
+    } = params;
     this.id = id;
     this.callId = callId;
     this.args = args;
     this.name = name;
     this.createdAt = createdAt;
+    this.extra = { ...extra };
+    this.groupId = groupId;
+    this.thoughtSignature =
+      thoughtSignature ??
+      (typeof this.extra.google === 'object' && this.extra.google !== null
+        ? // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          (this.extra.google as any).thoughtSignature ||
+          (this.extra.google as any).thought_signature
+        : undefined);
   }
 
   static create(params: {
@@ -210,6 +284,9 @@ export class FunctionCall {
     args: string;
     id?: string;
     createdAt?: number;
+    extra?: Record<string, unknown>;
+    groupId?: string;
+    thoughtSignature?: string;
   }) {
     return new FunctionCall(params);
   }
@@ -224,6 +301,18 @@ export class FunctionCall {
       args: this.args,
     };
 
+    if (Object.keys(this.extra).length > 0) {
+      result.extra = this.extra as JSONValue;
+    }
+
+    if (this.groupId) {
+      result.groupId = this.groupId;
+    }
+
+    if (this.thoughtSignature) {
+      result.thoughtSignature = this.thoughtSignature;
+    }
+
     if (!excludeTimestamp) {
       result.createdAt = this.createdAt;
     }
@@ -300,7 +389,60 @@ export class FunctionCallOutput {
   }
 }
 
-export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput;
+export class AgentHandoffItem {
+  readonly id: string;
+
+  readonly type = 'agent_handoff' as const;
+
+  oldAgentId: string | undefined;
+
+  newAgentId: string;
+
+  createdAt: number;
+
+  constructor(params: {
+    oldAgentId?: string;
+    newAgentId: string;
+    id?: string;
+    createdAt?: number;
+  }) {
+    const { oldAgentId, newAgentId, id = shortuuid('item_'), createdAt = Date.now() } = params;
+    this.id = id;
+    this.oldAgentId = oldAgentId;
+    this.newAgentId = newAgentId;
+    this.createdAt = createdAt;
+  }
+
+  static create(params: {
+    oldAgentId?: string;
+    newAgentId: string;
+    id?: string;
+    createdAt?: number;
+  }) {
+    return new AgentHandoffItem(params);
+  }
+
+  toJSON(excludeTimestamp: boolean = false): JSONValue {
+    const result: JSONValue = {
+      id: this.id,
+      type: this.type,
+      newAgentId: this.newAgentId,
+    };
+
+    if (this.oldAgentId !== undefined) {
+      result.oldAgentId = this.oldAgentId;
+    }
+
+    if (!excludeTimestamp) {
+      result.createdAt = this.createdAt;
+    }
+
+    return result;
+  }
+}
+
+// TODO(parity): Add AgentConfigUpdate type to ChatItem union
+export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput | AgentHandoffItem;
 
 export class ChatContext {
   protected _items: ChatItem[];
@@ -330,6 +472,9 @@ export class ChatContext {
     id?: string;
     interrupted?: boolean;
     createdAt?: number;
+    transcriptConfidence?: number;
+    metrics?: MetricsReport;
+    extra?: Record<string, unknown>;
   }): ChatMessage {
     const msg = new ChatMessage(params);
     if (params.createdAt !== undefined) {
@@ -362,11 +507,13 @@ export class ChatContext {
     return idx !== -1 ? idx : undefined;
   }
 
+  // TODO(parity): Add excludeConfigUpdate option when AgentConfigUpdate is ported
   copy(
     options: {
       excludeFunctionCall?: boolean;
       excludeInstructions?: boolean;
       excludeEmptyMessage?: boolean;
+      excludeHandoff?: boolean;
       toolCtx?: ToolContext<any>; // eslint-disable-line @typescript-eslint/no-explicit-any
     } = {},
   ): ChatContext {
@@ -374,6 +521,7 @@ export class ChatContext {
       excludeFunctionCall = false,
       excludeInstructions = false,
       excludeEmptyMessage = false,
+      excludeHandoff = false,
       toolCtx,
     } = options;
     const items: ChatItem[] = [];
@@ -399,6 +547,10 @@ export class ChatContext {
         continue;
       }
 
+      if (excludeHandoff && item.type === 'agent_handoff') {
+        continue;
+      }
+
       if (toolCtx !== undefined && isToolCallOrOutput(item) && toolCtx[item.name] === undefined) {
         continue;
       }
@@ -409,6 +561,42 @@ export class ChatContext {
     return new ChatContext(items);
   }
 
+  // TODO(parity): Add excludeConfigUpdate option when AgentConfigUpdate is ported
+  merge(
+    other: ChatContext,
+    options: {
+      excludeFunctionCall?: boolean;
+      excludeInstructions?: boolean;
+    } = {},
+  ): ChatContext {
+    const { excludeFunctionCall = false, excludeInstructions = false } = options;
+    const existingIds = new Set(this._items.map((item) => item.id));
+
+    for (const item of other.items) {
+      if (excludeFunctionCall && ['function_call', 'function_call_output'].includes(item.type)) {
+        continue;
+      }
+
+      if (
+        excludeInstructions &&
+        item.type === 'message' &&
+        (item.role === 'system' || item.role === 'developer')
+      ) {
+        continue;
+      }
+
+      if (existingIds.has(item.id)) {
+        continue;
+      }
+
+      const idx = this.findInsertionIndex(item.createdAt);
+      this._items.splice(idx, 0, item);
+      existingIds.add(item.id);
+    }
+
+    return this;
+  }
+
   truncate(maxItems: number): ChatContext {
     if (maxItems <= 0) return this;
 
@@ -470,6 +658,9 @@ export class ChatContext {
           id: item.id,
           interrupted: item.interrupted,
           createdAt: item.createdAt,
+          transcriptConfidence: item.transcriptConfidence,
+          metrics: item.metrics,
+          extra: item.extra,
         });
 
         // Filter content based on options
@@ -513,6 +704,257 @@ export class ChatContext {
     return 0;
   }
 
+  /**
+   * Return true if `other` has the same sequence of items with matching
+   * essential fields (IDs, types, and payload) as this context.
+   *
+   * Comparison rules:
+   * - Messages: compares the full `content` list, `role` and `interrupted`.
+   * - Function calls: compares `name`, `callId`, and `args`.
+   * - Function call outputs: compares `name`, `callId`, `output`, and `isError`.
+   *
+   * Does not consider timestamps or other metadata.
+   */
+  isEquivalent(other: ChatContext): boolean {
+    if (this === other) {
+      return true;
+    }
+
+    if (this.items.length !== other.items.length) {
+      return false;
+    }
+
+    for (let i = 0; i < this.items.length; i++) {
+      const a = this.items[i]!;
+      const b = other.items[i]!;
+
+      if (a.id !== b.id || a.type !== b.type) {
+        return false;
+      }
+
+      if (a.type === 'message' && b.type === 'message') {
+        if (
+          a.role !== b.role ||
+          a.interrupted !== b.interrupted ||
+          !this.compareContent(a.content, b.content)
+        ) {
+          return false;
+        }
+      } else if (a.type === 'function_call' && b.type === 'function_call') {
+        if (
+          a.name !== b.name ||
+          a.callId !== b.callId ||
+          a.args !== b.args ||
+          a.thoughtSignature !== b.thoughtSignature ||
+          a.groupId !== b.groupId ||
+          JSON.stringify(a.extra) !== JSON.stringify(b.extra)
+        ) {
+          return false;
+        }
+      } else if (a.type === 'function_call_output' && b.type === 'function_call_output') {
+        if (
+          a.name !== b.name ||
+          a.callId !== b.callId ||
+          a.output !== b.output ||
+          a.isError !== b.isError
+        ) {
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * Compare two content arrays for equality.
+   */
+  private compareContent(a: ChatContent[], b: ChatContent[]): boolean {
+    if (a.length !== b.length) {
+      return false;
+    }
+
+    for (let i = 0; i < a.length; i++) {
+      const contentA = a[i]!;
+      const contentB = b[i]!;
+
+      if (typeof contentA === 'string' && typeof contentB === 'string') {
+        if (contentA !== contentB) {
+          return false;
+        }
+        continue;
+      }
+
+      if (typeof contentA !== typeof contentB) {
+        return false;
+      }
+
+      if (typeof contentA === 'object' && typeof contentB === 'object') {
+        if (contentA.type === 'image_content' && contentB.type === 'image_content') {
+          if (
+            contentA.id !== contentB.id ||
+            contentA.image !== contentB.image ||
+            contentA.inferenceDetail !== contentB.inferenceDetail ||
+            contentA.inferenceWidth !== contentB.inferenceWidth ||
+            contentA.inferenceHeight !== contentB.inferenceHeight ||
+            contentA.mimeType !== contentB.mimeType
+          ) {
+            return false;
+          }
+        } else if (contentA.type === 'audio_content' && contentB.type === 'audio_content') {
+          if (contentA.frame.length !== contentB.frame.length) {
+            return false;
+          }
+          if (contentA.transcript !== contentB.transcript) {
+            return false;
+          }
+        } else {
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
+
+  async _summarize(llm: LLM, options: { keepLastTurns?: number } = {}): Promise<ChatContext> {
+    const { keepLastTurns = 2 } = options;
+
+    // Split the history into a head/tail over the full item stream so recent
+    // tool calls/outputs stay attached to the turns they belong to.
+    const msgBudget = keepLastTurns * 2;
+    let splitIdx = this._items.length;
+
+    if (msgBudget > 0) {
+      let msgCount = 0;
+      let foundSplit = false;
+      for (let i = this._items.length - 1; i >= 0; i -= 1) {
+        const item = this._items[i]!;
+        if (item.type === 'message' && (item.role === 'user' || item.role === 'assistant')) {
+          msgCount += 1;
+          if (msgCount >= msgBudget) {
+            splitIdx = i;
+            foundSplit = true;
+            break;
+          }
+        }
+      }
+
+      if (!foundSplit) {
+        return this;
+      }
+    }
+
+    if (splitIdx === 0) {
+      return this;
+    }
+
+    const headItems = this._items.slice(0, splitIdx);
+    const tailItems = this._items.slice(splitIdx);
+
+    const toSummarize: Array<ChatMessage | FunctionCall | FunctionCallOutput> = [];
+    for (const item of headItems) {
+      if (item.type === 'message') {
+        if (item.role !== 'user' && item.role !== 'assistant') continue;
+        if (item.extra?.is_summary === true) continue;
+
+        const text = (item.textContent ?? '').trim();
+        if (text) {
+          toSummarize.push(item);
+        }
+      } else if (item.type === 'function_call' || item.type === 'function_call_output') {
+        toSummarize.push(item);
+      }
+    }
+
+    if (toSummarize.length === 0) {
+      return this;
+    }
+
+    const sourceText = toSummarize
+      .map((item) => {
+        if (item.type === 'message') {
+          return toXml(item.role, (item.textContent ?? '').trim());
+        }
+
+        return functionCallItemToMessage(item).textContent ?? '';
+      })
+      .join('\n')
+      .trim();
+
+    if (!sourceText) {
+      return this;
+    }
+
+    // TODO: refactor this into LLMStream.collect API.
+    const promptCtx = new ChatContext();
+    promptCtx.addMessage({
+      role: 'system',
+      content: [
+        'Compress older conversation history into a short, faithful summary.',
+        '',
+        'The conversation is formatted as XML. Here is how to read it:',
+        '- <user>...</user>  - something the user said.',
+        '- <assistant>...</assistant>  - something the assistant said.',
+        '- <function_call name="..." call_id="...">...</function_call>  - the assistant invoked an action.',
+        '- <function_call_output name="..." call_id="...">...</function_call_output>  - the result of that action. May contain <error>...</error> if it failed.',
+        '',
+        'Guidelines:',
+        '- Distill the information learned from function call outputs into the summary. Do not mention that a tool or function was called; just preserve the knowledge gained.',
+        '- Focus on user goals, constraints, decisions, key facts, preferences, entities, and any pending or unresolved tasks.',
+        '- Omit greetings, filler, and chit-chat.',
+        '- Be concise.',
+      ].join('\n'),
+    });
+    promptCtx.addMessage({
+      role: 'user',
+      content: `Conversation to summarize:\n\n${sourceText}`,
+    });
+
+    const chunks: string[] = [];
+    for await (const chunk of llm.chat({ chatCtx: promptCtx })) {
+      if (chunk.delta?.content) {
+        chunks.push(chunk.delta.content);
+      }
+    }
+
+    const summary = chunks.join('').trim();
+    if (!summary) {
+      return this;
+    }
+
+    const preserved: ChatItem[] = [];
+    for (const it of headItems) {
+      if (it.type === 'message' && (it.role === 'user' || it.role === 'assistant')) {
+        continue;
+      }
+
+      if (it.type === 'function_call' || it.type === 'function_call_output') {
+        continue;
+      }
+
+      preserved.push(it);
+    }
+
+    this._items = preserved;
+
+    const createdAtHint =
+      tailItems.length > 0
+        ? tailItems[0]!.createdAt - 1e-6
+        : headItems[headItems.length - 1]!.createdAt + 1e-6;
+
+    this.addMessage({
+      role: 'assistant',
+      content: toXml('chat_history_summary', summary),
+      createdAt: createdAtHint,
+      extra: { is_summary: true },
+    });
+
+    this._items.push(...tailItems);
+
+    return this;
+  }
+
   /**
    * Indicates whether the context is read-only
    */
@@ -521,6 +963,55 @@ export class ChatContext {
   }
 }
 
+function toAttrsStr(attrs?: Record<string, unknown>): string | undefined {
+  if (!attrs) {
+    return undefined;
+  }
+
+  return Object.entries(attrs)
+    .map(([key, value]) => `${key}="${String(value)}"`)
+    .join(' ');
+}
+
+function toXml(tagName: string, content?: string, attrs?: Record<string, unknown>): string {
+  const attrsStr = toAttrsStr(attrs);
+  if (content) {
+    return [attrsStr ? `<${tagName} ${attrsStr}>` : `<${tagName}>`, content, `</${tagName}>`].join(
+      '\n',
+    );
+  }
+
+  return attrsStr ? `<${tagName} ${attrsStr} />` : `<${tagName} />`;
+}
+
+function functionCallItemToMessage(item: FunctionCall | FunctionCallOutput): ChatMessage {
+  if (item.type === 'function_call') {
+    return new ChatMessage({
+      role: 'user',
+      content: [
+        toXml('function_call', item.args, {
+          name: item.name,
+          call_id: item.callId,
+        }),
+      ],
+      createdAt: item.createdAt,
+      extra: { is_function_call: true },
+    });
+  }
+
+  return new ChatMessage({
+    role: 'assistant',
+    content: [
+      toXml('function_call_output', item.isError ? toXml('error', item.output) : item.output, {
+        name: item.name,
+        call_id: item.callId,
+      }),
+    ],
+    createdAt: item.createdAt,
+    extra: { is_function_call_output: true },
+  });
+}
+
 export class ReadonlyChatContext extends ChatContext {
   static readonly errorMsg =
     'Please use .copy() and agent.update_chat_ctx() to modify the chat context.';