@livekit/agents 1.1.0 → 1.2.0

package/src/llm/provider_format/openai.test.ts

@@ -4,9 +4,14 @@
 import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { initializeLogger } from '../../log.js';
-import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js';
+import {
+  AgentHandoffItem,
+  ChatContext,
+  FunctionCall,
+  FunctionCallOutput,
+} from '../chat_context.js';
 import { serializeImage } from '../utils.js';
-import { toChatCtx } from './openai.js';
+import { toChatCtx, toResponsesChatCtx } from './openai.js';
 
 // Mock the serializeImage function
 vi.mock('../utils.js', () => ({
@@ -253,6 +258,46 @@ describe('toChatCtx', () => {
     ]);
   });
 
+  it('should include provider-specific extra content on tool calls', async () => {
+    const ctx = ChatContext.empty();
+    const msg = ctx.addMessage({ role: 'assistant', content: 'Running tool' });
+
+    const toolCall = FunctionCall.create({
+      id: `${msg.id}/tool_1`,
+      callId: 'call_789',
+      name: 'google_call',
+      args: '{}',
+      extra: { google: { thoughtSignature: 'sig-123' } },
+    });
+    const toolOutput = FunctionCallOutput.create({
+      callId: 'call_789',
+      output: '{"result": "ok"}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall, toolOutput]);
+
+    const result = await toChatCtx(ctx);
+
+    expect(result[0]).toEqual({
+      role: 'assistant',
+      content: 'Running tool',
+      tool_calls: [
+        {
+          type: 'function',
+          id: 'call_789',
+          function: { name: 'google_call', arguments: '{}' },
+          extra_content: { google: { thoughtSignature: 'sig-123' } },
+        },
+      ],
+    });
+    expect(result[1]).toEqual({
+      role: 'tool',
+      tool_call_id: 'call_789',
+      content: '{"result": "ok"}',
+    });
+  });
+
   it('should handle multiple tool calls in one message', async () => {
     const ctx = ChatContext.empty();
 
@@ -578,4 +623,437 @@ describe('toChatCtx', () => {
       },
     ]);
   });
+
+  it('should filter out agent handoff items', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+
+    // Insert an agent handoff item
+    const handoff = new AgentHandoffItem({
+      oldAgentId: 'agent_1',
+      newAgentId: 'agent_2',
+    });
+    ctx.insert(handoff);
+
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const result = await toChatCtx(ctx);
+
+    // Agent handoff should be filtered out, only messages should remain
+    expect(result).toEqual([
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there!' },
+    ]);
+  });
+
+  it('should handle multiple agent handoffs without errors', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Start' });
+
+    // Multiple handoffs
+    ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' });
+
+    const result = await toChatCtx(ctx);
+
+    // All handoffs should be filtered out
+    expect(result).toEqual([
+      { role: 'user', content: 'Start' },
+      { role: 'assistant', content: 'Response from agent 1' },
+      { role: 'assistant', content: 'Response from agent 2' },
+      { role: 'assistant', content: 'Response from agent 3' },
+    ]);
+  });
+});
+
+describe('toResponsesChatCtx', () => {
+  const serializeImageMock = vi.mocked(serializeImage);
+
+  initializeLogger({ level: 'silent', pretty: false });
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+  });
+
+  it('should convert simple text messages', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(2);
+    expect(result[0]).toEqual({ role: 'user', content: 'Hello' });
+    expect(result[1]).toEqual({ role: 'assistant', content: 'Hi there!' });
+  });
+
+  it('should handle system messages', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'system', content: 'You are a helpful assistant' });
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(2);
+    expect(result[0]).toEqual({ role: 'system', content: 'You are a helpful assistant' });
+    expect(result[1]).toEqual({ role: 'user', content: 'Hello' });
+  });
+
+  it('should handle multi-line text content', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: ['Line 1', 'Line 2', 'Line 3'] });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(1);
+    expect(result[0]).toEqual({ role: 'user', content: 'Line 1\nLine 2\nLine 3' });
+  });
+
+  it('should convert images to input_image format with external URL', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      externalUrl: 'https://example.com/image.jpg',
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'https://example.com/image.jpg',
+          inferenceDetail: 'high',
+          _cache: {},
+        },
+      ],
+    });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'input_image',
+            image_url: 'https://example.com/image.jpg',
+            detail: 'high',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('should convert images to input_image format with base64 data', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'auto',
+      mimeType: 'image/png',
+      base64Data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB',
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB',
+          inferenceDetail: 'auto',
+          _cache: {},
+        },
+      ],
+    });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'input_image',
+            image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB',
+            detail: 'auto',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('should handle mixed content with text and image using input_text', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      externalUrl: 'https://example.com/image.jpg',
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        'Check this out:',
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'https://example.com/image.jpg',
+          inferenceDetail: 'high',
+          _cache: {},
+        },
+      ],
+    });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'input_image',
+            image_url: 'https://example.com/image.jpg',
+            detail: 'high',
+          },
+          { type: 'input_text', text: 'Check this out:' },
+        ],
+      },
+    ]);
+  });
+
+  it('should handle tool calls as top-level function_call items', async () => {
+    const ctx = ChatContext.empty();
+
+    const msg = ctx.addMessage({ role: 'assistant', content: 'Let me help you.' });
+    const toolCall = FunctionCall.create({
+      id: msg.id + '/tool_1',
+      callId: 'call_123',
+      name: 'get_weather',
+      args: '{"location": "Paris"}',
+    });
+    const toolOutput = FunctionCallOutput.create({
+      callId: 'call_123',
+      output: '{"temperature": 20}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall, toolOutput]);
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      { role: 'assistant', content: 'Let me help you.' },
+      {
+        type: 'function_call',
+        call_id: 'call_123',
+        name: 'get_weather',
+        arguments: '{"location": "Paris"}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_123',
+        output: '{"temperature": 20}',
+      },
+    ]);
+  });
+
+  it('should handle tool calls without an accompanying message', async () => {
+    const ctx = ChatContext.empty();
+
+    const toolCall = new FunctionCall({
+      id: 'func_1',
+      callId: 'call_456',
+      name: 'calculate',
+      args: '{"a": 5, "b": 3}',
+    });
+    const toolOutput = new FunctionCallOutput({
+      callId: 'call_456',
+      output: '{"result": 8}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall, toolOutput]);
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        type: 'function_call',
+        call_id: 'call_456',
+        name: 'calculate',
+        arguments: '{"a": 5, "b": 3}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_456',
+        output: '{"result": 8}',
+      },
+    ]);
+  });
+
+  it('should handle multiple tool calls as separate function_call items', async () => {
+    const ctx = ChatContext.empty();
+
+    const msg = ctx.addMessage({ role: 'assistant', content: "I'll check both." });
+    const toolCall1 = new FunctionCall({
+      id: msg.id + '/tool_1',
+      callId: 'call_1',
+      name: 'get_weather',
+      args: '{"location": "NYC"}',
+    });
+    const toolCall2 = new FunctionCall({
+      id: msg.id + '/tool_2',
+      callId: 'call_2',
+      name: 'get_weather',
+      args: '{"location": "LA"}',
+    });
+    const toolOutput1 = new FunctionCallOutput({
+      callId: 'call_1',
+      output: '{"temperature": 65}',
+      isError: false,
+    });
+    const toolOutput2 = new FunctionCallOutput({
+      callId: 'call_2',
+      output: '{"temperature": 78}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall1, toolCall2, toolOutput1, toolOutput2]);
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      { role: 'assistant', content: "I'll check both." },
+      {
+        type: 'function_call',
+        call_id: 'call_1',
+        name: 'get_weather',
+        arguments: '{"location": "NYC"}',
+      },
+      {
+        type: 'function_call',
+        call_id: 'call_2',
+        name: 'get_weather',
+        arguments: '{"location": "LA"}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_1',
+        output: '{"temperature": 65}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_2',
+        output: '{"temperature": 78}',
+      },
+    ]);
+  });
+
+  it('should skip empty groups', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: 'Hello', createdAt: 1000 });
+
+    const orphanOutput = new FunctionCallOutput({
+      callId: 'orphan_call',
+      output: 'This should be ignored',
+      isError: false,
+      createdAt: 2000,
+    });
+    ctx.insert(orphanOutput);
+
+    ctx.addMessage({ role: 'assistant', content: 'Hi!', createdAt: 3000 });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(2);
+    expect(result).toContainEqual({ role: 'user', content: 'Hello' });
+    expect(result).toContainEqual({ role: 'assistant', content: 'Hi!' });
+  });
+
+  it('should filter out agent handoff items', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there!' },
+    ]);
+  });
+
+  it('should cache serialized images', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      mimeType: 'image/png',
+      base64Data: 'cached-data',
+    });
+
+    const imageContent = {
+      id: 'img1',
+      type: 'image_content' as const,
+      image: 'https://example.com/image.jpg',
+      inferenceDetail: 'high' as const,
+      _cache: {},
+    };
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: [imageContent] });
+
+    await toResponsesChatCtx(ctx);
+    await toResponsesChatCtx(ctx);
+
+    expect(serializeImageMock).toHaveBeenCalledTimes(1);
+    expect(imageContent._cache).toHaveProperty('serialized_image');
+  });
+
+  it('should throw error for unsupported content type', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          type: 'audio_content',
+          frame: [],
+        },
+      ],
+    });
+
+    await expect(toResponsesChatCtx(ctx)).rejects.toThrow(
+      'Unsupported content type: audio_content',
+    );
+  });
+
+  it('should throw error when serialized image has no data', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      // No base64Data or externalUrl
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'invalid-image',
+          inferenceDetail: 'high',
+          _cache: {},
+        },
+      ],
+    });
+
+    await expect(toResponsesChatCtx(ctx)).rejects.toThrow('Serialized image has no data bytes');
+  });
 });

package/src/llm/provider_format/openai.ts

@@ -17,11 +17,20 @@ export async function toChatCtx(chatCtx: ChatContext, injectDummyUserMessage: bo
       ? await toChatItem(group.message)
       : { role: 'assistant' };
 
-    const toolCalls = group.toolCalls.map((toolCall) => ({
-      type: 'function',
-      id: toolCall.callId,
-      function: { name: toolCall.name, arguments: toolCall.args },
-    }));
+    const toolCalls = group.toolCalls.map((toolCall) => {
+      const tc: Record<string, any> = {
+        type: 'function',
+        id: toolCall.callId,
+        function: { name: toolCall.name, arguments: toolCall.args },
+      };
+
+      // Include provider-specific extra content (e.g., Google thought signatures)
+      const googleExtra = getGoogleExtra(toolCall);
+      if (googleExtra) {
+        tc.extra_content = { google: googleExtra };
+      }
+      return tc;
+    });
 
     if (toolCalls.length > 0) {
       message['tool_calls'] = toolCalls;
@@ -53,24 +62,33 @@ async function toChatItem(item: ChatItem) {
       }
     }
 
-    const content =
-      listContent.length == 0
-        ? textContent
-        : textContent.length == 0
-          ? listContent
-          : [...listContent, { type: 'text', text: textContent }];
+    const result: Record<string, any> = { role: item.role };
+    if (listContent.length === 0) {
+      result.content = textContent;
+    } else {
+      if (textContent.length > 0) {
+        listContent.push({ type: 'text', text: textContent });
+      }
+      result.content = listContent;
+    }
 
-    return { role: item.role, content };
+    return result;
   } else if (item.type === 'function_call') {
+    const tc: Record<string, any> = {
+      id: item.callId,
+      type: 'function',
+      function: { name: item.name, arguments: item.args },
+    };
+
+    // Include provider-specific extra content (e.g., Google thought signatures)
+    const googleExtra = getGoogleExtra(item);
+    if (googleExtra) {
+      tc.extra_content = { google: googleExtra };
+    }
+
     return {
       role: 'assistant',
-      tool_calls: [
-        {
-          id: item.callId,
-          type: 'function',
-          function: { name: item.name, arguments: item.args },
-        },
-      ],
+      tool_calls: [tc],
     };
   } else if (item.type === 'function_call_output') {
     return {
@@ -78,9 +96,19 @@ async function toChatItem(item: ChatItem) {
       tool_call_id: item.callId,
       content: item.output,
     };
-  } else {
-    throw new Error(`Unsupported item type: ${item['type']}`);
   }
+  // Skip other item types (e.g., agent_handoff)
+  // These should be filtered by groupToolCalls, but this is a safety net
+  throw new Error(`Unsupported item type: ${item['type']}`);
+}
+
+function getGoogleExtra(
+  item: Partial<{ extra?: Record<string, unknown>; thoughtSignature?: string }>,
+): Record<string, unknown> | undefined {
+  const googleExtra =
+    (item.extra?.google as Record<string, unknown> | undefined) ||
+    (item.thoughtSignature ? { thoughtSignature: item.thoughtSignature } : undefined);
+  return googleExtra;
 }
 
 async function toImageContent(content: ImageContent) {
@@ -116,3 +144,106 @@ async function toImageContent(content: ImageContent) {
     },
   };
 }
+
+async function toResponsesImageContent(content: ImageContent) {
+  const cacheKey = 'serialized_image';
+  let serialized: SerializedImage;
+
+  if (content._cache[cacheKey] === undefined) {
+    serialized = await serializeImage(content);
+    content._cache[cacheKey] = serialized;
+  }
+  serialized = content._cache[cacheKey];
+
+  if (serialized.externalUrl) {
+    return {
+      type: 'input_image' as const,
+      image_url: serialized.externalUrl,
+      detail: serialized.inferenceDetail,
+    };
+  }
+
+  if (serialized.base64Data === undefined) {
+    throw new Error('Serialized image has no data bytes');
+  }
+
+  return {
+    type: 'input_image' as const,
+    image_url: `data:${serialized.mimeType};base64,${serialized.base64Data}`,
+    detail: serialized.inferenceDetail,
+  };
+}
+
+export async function toResponsesChatCtx(
+  chatCtx: ChatContext,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  injectDummyUserMessage: boolean = true,
+) {
+  const itemGroups = groupToolCalls(chatCtx);
+  const messages: Record<string, any>[] = []; // eslint-disable-line @typescript-eslint/no-explicit-any
+
+  for (const group of itemGroups) {
+    if (group.isEmpty) continue;
+
+    if (group.message) {
+      messages.push(await toResponsesChatItem(group.message));
+    }
+
+    for (const toolCall of group.toolCalls) {
+      messages.push({
+        type: 'function_call',
+        call_id: toolCall.callId,
+        name: toolCall.name,
+        arguments: toolCall.args,
+      });
+    }
+
+    for (const toolOutput of group.toolOutputs) {
+      messages.push(await toResponsesChatItem(toolOutput));
+    }
+  }
+
+  return messages;
+}
+
+async function toResponsesChatItem(item: ChatItem) {
+  if (item.type === 'message') {
+    const listContent: Record<string, any>[] = []; // eslint-disable-line @typescript-eslint/no-explicit-any
+    let textContent = '';
+
+    for (const content of item.content) {
+      if (typeof content === 'string') {
+        if (textContent) textContent += '\n';
+        textContent += content;
+      } else if (content.type === 'image_content') {
+        listContent.push(await toResponsesImageContent(content));
+      } else {
+        throw new Error(`Unsupported content type: ${content.type}`);
+      }
+    }
+
+    const content =
+      listContent.length == 0
+        ? textContent
+        : textContent.length == 0
+          ? listContent
+          : [...listContent, { type: 'input_text', text: textContent }];
+
+    return { role: item.role, content };
+  } else if (item.type === 'function_call') {
+    return {
+      type: 'function_call',
+      call_id: item.callId,
+      name: item.name,
+      arguments: item.args,
+    };
+  } else if (item.type === 'function_call_output') {
+    return {
+      type: 'function_call_output',
+      call_id: item.callId,
+      output: item.output,
+    };
+  }
+
+  throw new Error(`Unsupported item type: ${item['type']}`);
+}