@livekit/agents 1.2.0 → 1.2.1

package/src/llm/chat_context.test.ts

@@ -17,6 +17,18 @@ import {
 
 initializeLogger({ pretty: false, level: 'error' });
 
+const summaryXml = (summary: string) =>
+  ['<chat_history_summary>', summary, '</chat_history_summary>'].join('\n');
+
+class TrackingFakeLLM extends FakeLLM {
+  chatCalls = 0;
+
+  chat(...args: Parameters<FakeLLM['chat']>) {
+    this.chatCalls += 1;
+    return super.chat(...args);
+  }
+}
+
 describe('ChatContext.toJSON', () => {
   it('should match snapshot for empty context', () => {
     const context = new ChatContext();
@@ -288,18 +300,26 @@ describe('ChatContext.toJSON', () => {
 });
 
 describe('ChatContext._summarize', () => {
-  it('keeps chronological timestamps with summary + tail', async () => {
+  it('includes function calls in the summarization source and keeps chronological order', async () => {
     const ctx = new ChatContext();
     ctx.addMessage({ role: 'system', content: 'System prompt', createdAt: 0 });
     ctx.addMessage({ role: 'user', content: 'hello', createdAt: 1000 });
     ctx.addMessage({ role: 'assistant', content: 'hi there', createdAt: 2000 });
+    ctx.insert(
+      FunctionCall.create({
+        callId: 'call_1',
+        name: 'lookup',
+        args: '{"order":"123"}',
+        createdAt: 2500,
+      }),
+    );
     ctx.insert(
       new FunctionCallOutput({
         callId: 'call_1',
         name: 'lookup',
-        output: '{"ok":true}',
+        output: '{"status":"delivered"}',
         isError: false,
-        createdAt: 3500,
+        createdAt: 2600,
       }),
     );
     ctx.addMessage({ role: 'user', content: 'my color is blue', createdAt: 3000 });
@@ -307,7 +327,22 @@ describe('ChatContext._summarize', () => {
 
     const fake = new FakeLLM([
       {
-        input: 'Conversation to summarize:\n\nuser: hello\nassistant: hi there',
+        input: [
+          'Conversation to summarize:',
+          '',
+          '<user>',
+          'hello',
+          '</user>',
+          '<assistant>',
+          'hi there',
+          '</assistant>',
+          '<function_call name="lookup" call_id="call_1">',
+          '{"order":"123"}',
+          '</function_call>',
+          '<function_call_output name="lookup" call_id="call_1">',
+          '{"status":"delivered"}',
+          '</function_call_output>',
+        ].join('\n'),
         content: 'condensed head',
       },
     ]);
@@ -323,12 +358,109 @@ describe('ChatContext._summarize', () => {
       throw new Error('summary message is missing');
     }
 
-    expect(summary.createdAt).toBeCloseTo(2999.999, 6);
+    expect(summary.textContent).toBe(summaryXml('condensed head'));
+    expect(summary.createdAt).toBeCloseTo(2999.999999, 6);
+    expect(ctx.items.filter((item) => item.type === 'function_call')).toHaveLength(0);
+    expect(ctx.items.filter((item) => item.type === 'function_call_output')).toHaveLength(0);
 
     const createdAts = ctx.items.map((item) => item.createdAt);
     const sorted = [...createdAts].sort((a, b) => a - b);
     expect(createdAts).toEqual(sorted);
   });
+
+  it('preserves interleaved tool items that belong to the recent tail', async () => {
+    const ctx = new ChatContext();
+    ctx.addMessage({ role: 'system', content: 'System prompt', createdAt: 0 });
+    ctx.addMessage({ role: 'user', content: 'my earbuds are broken', createdAt: 1000 });
+    ctx.addMessage({
+      role: 'assistant',
+      content: 'Can you share your order number?',
+      createdAt: 2000,
+    });
+    ctx.addMessage({ role: 'user', content: 'Order #123', createdAt: 3000 });
+    ctx.insert(
+      FunctionCall.create({
+        callId: 'call_2',
+        name: 'lookup_order',
+        args: '{"order":"123"}',
+        createdAt: 3500,
+      }),
+    );
+    ctx.insert(
+      new FunctionCallOutput({
+        callId: 'call_2',
+        name: 'lookup_order',
+        output: '{"status":"delivered"}',
+        isError: false,
+        createdAt: 3600,
+      }),
+    );
+    ctx.addMessage({
+      role: 'assistant',
+      content: 'Found your order. Let me check the warranty.',
+      createdAt: 4000,
+    });
+    ctx.addMessage({ role: 'user', content: 'Thanks.', createdAt: 5000 });
+    ctx.addMessage({ role: 'assistant', content: 'You are under warranty.', createdAt: 6000 });
+
+    const fake = new FakeLLM([
+      {
+        input: [
+          'Conversation to summarize:',
+          '',
+          '<user>',
+          'my earbuds are broken',
+          '</user>',
+          '<assistant>',
+          'Can you share your order number?',
+          '</assistant>',
+        ].join('\n'),
+        content: 'older summary',
+      },
+    ]);
+
+    await ctx._summarize(fake, { keepLastTurns: 2 });
+
+    const functionItems = ctx.items.filter(
+      (item) => item.type === 'function_call' || item.type === 'function_call_output',
+    );
+    expect(functionItems).toHaveLength(2);
+    expect(functionItems.map((item) => item.createdAt)).toEqual([3500, 3600]);
+
+    const rawTailMessages = ctx.items.filter(
+      (item) =>
+        item.type === 'message' &&
+        (item.role === 'user' || item.role === 'assistant') &&
+        item.extra?.is_summary !== true,
+    );
+    expect(rawTailMessages).toHaveLength(4);
+    expect(rawTailMessages.map((item) => item.textContent)).toEqual([
+      'Order #123',
+      'Found your order. Let me check the warranty.',
+      'Thanks.',
+      'You are under warranty.',
+    ]);
+
+    const createdAts = ctx.items.map((item) => item.createdAt);
+    const sorted = [...createdAts].sort((a, b) => a - b);
+    expect(createdAts).toEqual(sorted);
+  });
+
+  it('skips summarization when the recent-turn budget already covers the history', async () => {
+    const ctx = new ChatContext();
+    ctx.addMessage({ role: 'system', content: 'System prompt', createdAt: 0 });
+    ctx.addMessage({ role: 'user', content: 'hello', createdAt: 1000 });
+    ctx.addMessage({ role: 'assistant', content: 'hi there', createdAt: 2000 });
+
+    const llm = new TrackingFakeLLM();
+    const originalIds = ctx.items.map((item) => item.id);
+
+    const result = await ctx._summarize(llm, { keepLastTurns: 1 });
+
+    expect(result).toBe(ctx);
+    expect(llm.chatCalls).toBe(0);
+    expect(ctx.items.map((item) => item.id)).toEqual(originalIds);
+  });
 });
 
 describe('ReadonlyChatContext with immutable array', () => {

package/src/llm/chat_context.ts

@@ -820,39 +820,65 @@ export class ChatContext {
   async _summarize(llm: LLM, options: { keepLastTurns?: number } = {}): Promise<ChatContext> {
     const { keepLastTurns = 2 } = options;
 
-    const toSummarize: ChatMessage[] = [];
-    for (const item of this._items) {
-      if (item.type !== 'message') continue;
-      if (item.role !== 'user' && item.role !== 'assistant') continue;
-      if (item.extra?.is_summary === true) continue;
+    // Split the history into a head/tail over the full item stream so recent
+    // tool calls/outputs stay attached to the turns they belong to.
+    const msgBudget = keepLastTurns * 2;
+    let splitIdx = this._items.length;
+
+    if (msgBudget > 0) {
+      let msgCount = 0;
+      let foundSplit = false;
+      for (let i = this._items.length - 1; i >= 0; i -= 1) {
+        const item = this._items[i]!;
+        if (item.type === 'message' && (item.role === 'user' || item.role === 'assistant')) {
+          msgCount += 1;
+          if (msgCount >= msgBudget) {
+            splitIdx = i;
+            foundSplit = true;
+            break;
+          }
+        }
+      }
 
-      const text = (item.textContent ?? '').trim();
-      if (text) {
-        toSummarize.push(item);
+      if (!foundSplit) {
+        return this;
       }
     }
 
-    if (toSummarize.length === 0) {
+    if (splitIdx === 0) {
       return this;
     }
 
-    const tailN = Math.max(0, Math.min(toSummarize.length, keepLastTurns * 2));
-    let head: ChatMessage[];
-    let tail: ChatMessage[];
-    if (tailN === 0) {
-      head = toSummarize;
-      tail = [];
-    } else {
-      head = toSummarize.slice(0, -tailN);
-      tail = toSummarize.slice(-tailN);
+    const headItems = this._items.slice(0, splitIdx);
+    const tailItems = this._items.slice(splitIdx);
+
+    const toSummarize: Array<ChatMessage | FunctionCall | FunctionCallOutput> = [];
+    for (const item of headItems) {
+      if (item.type === 'message') {
+        if (item.role !== 'user' && item.role !== 'assistant') continue;
+        if (item.extra?.is_summary === true) continue;
+
+        const text = (item.textContent ?? '').trim();
+        if (text) {
+          toSummarize.push(item);
+        }
+      } else if (item.type === 'function_call' || item.type === 'function_call_output') {
+        toSummarize.push(item);
+      }
     }
 
-    if (head.length === 0) {
+    if (toSummarize.length === 0) {
       return this;
     }
 
-    const sourceText = head
-      .map((m) => `${m.role}: ${(m.textContent ?? '').trim()}`)
+    const sourceText = toSummarize
+      .map((item) => {
+        if (item.type === 'message') {
+          return toXml(item.role, (item.textContent ?? '').trim());
+        }
+
+        return functionCallItemToMessage(item).textContent ?? '';
+      })
       .join('\n')
       .trim();
 
@@ -864,10 +890,21 @@ export class ChatContext {
     const promptCtx = new ChatContext();
     promptCtx.addMessage({
       role: 'system',
-      content:
-        'Compress older chat history into a short, faithful summary.\n' +
-        'Focus on user goals, constraints, decisions, key facts/preferences/entities, and pending tasks.\n' +
-        'Exclude chit-chat and greetings. Be concise.',
+      content: [
+        'Compress older conversation history into a short, faithful summary.',
+        '',
+        'The conversation is formatted as XML. Here is how to read it:',
+        '- <user>...</user>  - something the user said.',
+        '- <assistant>...</assistant>  - something the assistant said.',
+        '- <function_call name="..." call_id="...">...</function_call>  - the assistant invoked an action.',
+        '- <function_call_output name="..." call_id="...">...</function_call_output>  - the result of that action. May contain <error>...</error> if it failed.',
+        '',
+        'Guidelines:',
+        '- Distill the information learned from function call outputs into the summary. Do not mention that a tool or function was called; just preserve the knowledge gained.',
+        '- Focus on user goals, constraints, decisions, key facts, preferences, entities, and any pending or unresolved tasks.',
+        '- Omit greetings, filler, and chit-chat.',
+        '- Be concise.',
+      ].join('\n'),
     });
     promptCtx.addMessage({
       role: 'user',
@@ -886,18 +923,13 @@ export class ChatContext {
       return this;
     }
 
-    const tailStartTs = tail.length > 0 ? tail[0]!.createdAt : Infinity;
-
     const preserved: ChatItem[] = [];
-    for (const it of this._items) {
-      if (
-        (it.type === 'function_call' || it.type === 'function_call_output') &&
-        it.createdAt < tailStartTs
-      ) {
+    for (const it of headItems) {
+      if (it.type === 'message' && (it.role === 'user' || it.role === 'assistant')) {
         continue;
       }
 
-      if (it.type === 'message' && (it.role === 'user' || it.role === 'assistant')) {
+      if (it.type === 'function_call' || it.type === 'function_call_output') {
         continue;
       }
 
@@ -907,18 +939,18 @@ export class ChatContext {
     this._items = preserved;
 
     const createdAtHint =
-      tail.length > 0 ? tail[0]!.createdAt - 1e-3 : head[head.length - 1]!.createdAt + 1e-3;
+      tailItems.length > 0
+        ? tailItems[0]!.createdAt - 1e-6
+        : headItems[headItems.length - 1]!.createdAt + 1e-6;
 
     this.addMessage({
       role: 'assistant',
-      content: `[history summary]\n${summary}`,
+      content: toXml('chat_history_summary', summary),
       createdAt: createdAtHint,
       extra: { is_summary: true },
     });
 
-    for (const msg of tail) {
-      this.insert(msg);
-    }
+    this._items.push(...tailItems);
 
     return this;
   }
@@ -931,6 +963,55 @@ export class ChatContext {
   }
 }
 
+function toAttrsStr(attrs?: Record<string, unknown>): string | undefined {
+  if (!attrs) {
+    return undefined;
+  }
+
+  return Object.entries(attrs)
+    .map(([key, value]) => `${key}="${String(value)}"`)
+    .join(' ');
+}
+
+function toXml(tagName: string, content?: string, attrs?: Record<string, unknown>): string {
+  const attrsStr = toAttrsStr(attrs);
+  if (content) {
+    return [attrsStr ? `<${tagName} ${attrsStr}>` : `<${tagName}>`, content, `</${tagName}>`].join(
+      '\n',
+    );
+  }
+
+  return attrsStr ? `<${tagName} ${attrsStr} />` : `<${tagName} />`;
+}
+
+function functionCallItemToMessage(item: FunctionCall | FunctionCallOutput): ChatMessage {
+  if (item.type === 'function_call') {
+    return new ChatMessage({
+      role: 'user',
+      content: [
+        toXml('function_call', item.args, {
+          name: item.name,
+          call_id: item.callId,
+        }),
+      ],
+      createdAt: item.createdAt,
+      extra: { is_function_call: true },
+    });
+  }
+
+  return new ChatMessage({
+    role: 'assistant',
+    content: [
+      toXml('function_call_output', item.isError ? toXml('error', item.output) : item.output, {
+        name: item.name,
+        call_id: item.callId,
+      }),
+    ],
+    createdAt: item.createdAt,
+    extra: { is_function_call_output: true },
+  });
+}
+
 export class ReadonlyChatContext extends ChatContext {
   static readonly errorMsg =
     'Please use .copy() and agent.update_chat_ctx() to modify the chat context.';

package/src/llm/fallback_adapter.ts

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { Throws } from '@livekit/throws-transformer/throws';
 import { APIConnectionError, APIError } from '../_exceptions.js';
 import { log } from '../log.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
@@ -186,11 +187,12 @@ class FallbackLLMStream extends LLMStream {
   /**
    * Try to generate with a single LLM.
    * Returns an async generator that yields chunks.
+   * @throws {APIError} When the LLM returns an API error (retryable or not)
    */
   private async *tryGenerate(
     llm: LLM,
     checkRecovery: boolean = false,
-  ): AsyncGenerator<ChatChunk, void, unknown> {
+  ): AsyncGenerator<Throws<ChatChunk, APIError>, void, unknown> {
     const connOptions: APIConnectOptions = {
       ...this.connOptions,
       maxRetry: this.adapter.maxRetryPerLLM,
@@ -296,8 +298,9 @@ class FallbackLLMStream extends LLMStream {
 
   /**
    * Main run method - iterates through LLMs with fallback logic.
+   * @throws {APIConnectionError} When all LLM providers have been exhausted
    */
-  protected async run(): Promise<void> {
+  protected async run(): Promise<Throws<void, APIConnectionError>> {
     const startTime = Date.now();
 
     // Check if all LLMs are unavailable

package/src/llm/index.ts

@@ -63,10 +63,12 @@ export {
   computeChatCtxDiff,
   createToolOptions,
   executeToolCall,
+  formatChatHistory,
   oaiBuildFunctionInfo,
   oaiParams,
   serializeImage,
   toJsonSchema,
+  type FormatChatHistoryOptions,
   type OpenAIFunctionParameters,
   type SerializedImage,
 } from './utils.js';

package/src/llm/utils.test.ts

@@ -4,8 +4,15 @@
 import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
 import sharp from 'sharp';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
-import { ChatContext, ChatMessage, type ImageContent } from './chat_context.js';
-import { computeChatCtxDiff, serializeImage } from './utils.js';
+import {
+  AgentHandoffItem,
+  ChatContext,
+  ChatMessage,
+  FunctionCall,
+  FunctionCallOutput,
+  type ImageContent,
+} from './chat_context.js';
+import { computeChatCtxDiff, formatChatHistory, serializeImage } from './utils.js';
 
 function createChatMessage(
   id: string,
@@ -356,6 +363,100 @@ describe('computeChatCtxDiff', () => {
   });
 });
 
+describe('formatChatHistory', () => {
+  it('should format mixed chat history items for logging', () => {
+    const ctx = new ChatContext([
+      ChatMessage.create({
+        id: 'msg_system',
+        role: 'system',
+        content: 'You are helpful.',
+        createdAt: 1,
+      }),
+      ChatMessage.create({
+        id: 'msg_user',
+        role: 'user',
+        content: [
+          'Show me my order',
+          createImageContent('https://example.com/receipt.png'),
+          { type: 'audio_content', frame: [], transcript: 'order 123' },
+        ],
+        createdAt: 2,
+      }),
+      FunctionCall.create({
+        id: 'fn_call',
+        callId: 'call_1',
+        name: 'lookup_order',
+        args: '{"orderId":"123"}',
+        createdAt: 3,
+      }),
+      FunctionCallOutput.create({
+        id: 'fn_output',
+        callId: 'call_1',
+        name: 'lookup_order',
+        output: '{"status":"shipped","ok":true}',
+        isError: false,
+        createdAt: 4,
+      }),
+      AgentHandoffItem.create({
+        id: 'handoff',
+        oldAgentId: 'support',
+        newAgentId: 'returns',
+        createdAt: 5,
+      }),
+    ]);
+
+    expect(formatChatHistory(ctx)).toBe(
+      [
+        'Chat history (5 items)',
+        '',
+        '[0] message system',
+        '  You are helpful.',
+        '',
+        '[1] message user',
+        '  Show me my order',
+        '  [image url=https://example.com/receipt.png]',
+        '  [audio transcript="order 123"]',
+        '',
+        '[2] function_call lookup_order call_id=call_1',
+        '  {',
+        '    "orderId": "123"',
+        '  }',
+        '',
+        '[3] function_call_output lookup_order call_id=call_1',
+        '  {',
+        '    "status": "shipped",',
+        '    "ok": true',
+        '  }',
+        '',
+        '[4] agent_handoff',
+        '  support -> returns',
+      ].join('\n'),
+    );
+  });
+
+  it('should optionally include ids and timestamps', () => {
+    const ctx = new ChatContext([
+      FunctionCallOutput.create({
+        id: 'fn_output',
+        callId: 'call_1',
+        name: 'lookup_order',
+        output: 'plain text error',
+        isError: true,
+        createdAt: 123.456,
+      }),
+    ]);
+
+    expect(formatChatHistory(ctx, { includeIds: true, includeTimestamps: true })).toBe(
+      [
+        'Chat history (1 items)',
+        '',
+        '[0] function_call_output lookup_order call_id=call_1 error=true id=fn_output created_at=123.456',
+        '  plain text error',
+      ].join('\n'),
+    );
+  });
+});
+
 describe('serializeImage', () => {
   let consoleWarnSpy: ReturnType<typeof vi.spyOn>;