@livekit/agents 1.0.46 → 1.0.48

package/src/llm/provider_format/openai.test.ts

@@ -11,7 +11,7 @@ import {
   FunctionCallOutput,
 } from '../chat_context.js';
 import { serializeImage } from '../utils.js';
-import { toChatCtx } from './openai.js';
+import { toChatCtx, toResponsesChatCtx } from './openai.js';
 
 // Mock the serializeImage function
 vi.mock('../utils.js', () => ({
@@ -673,3 +673,387 @@ describe('toChatCtx', () => {
     ]);
   });
 });
+
+describe('toResponsesChatCtx', () => {
+  const serializeImageMock = vi.mocked(serializeImage);
+
+  initializeLogger({ level: 'silent', pretty: false });
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+  });
+
+  it('should convert simple text messages', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(2);
+    expect(result[0]).toEqual({ role: 'user', content: 'Hello' });
+    expect(result[1]).toEqual({ role: 'assistant', content: 'Hi there!' });
+  });
+
+  it('should handle system messages', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'system', content: 'You are a helpful assistant' });
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(2);
+    expect(result[0]).toEqual({ role: 'system', content: 'You are a helpful assistant' });
+    expect(result[1]).toEqual({ role: 'user', content: 'Hello' });
+  });
+
+  it('should handle multi-line text content', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: ['Line 1', 'Line 2', 'Line 3'] });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(1);
+    expect(result[0]).toEqual({ role: 'user', content: 'Line 1\nLine 2\nLine 3' });
+  });
+
+  it('should convert images to input_image format with external URL', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      externalUrl: 'https://example.com/image.jpg',
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'https://example.com/image.jpg',
+          inferenceDetail: 'high',
+          _cache: {},
+        },
+      ],
+    });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'input_image',
+            image_url: 'https://example.com/image.jpg',
+            detail: 'high',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('should convert images to input_image format with base64 data', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'auto',
+      mimeType: 'image/png',
+      base64Data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB',
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB',
+          inferenceDetail: 'auto',
+          _cache: {},
+        },
+      ],
+    });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'input_image',
+            image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB',
+            detail: 'auto',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('should handle mixed content with text and image using input_text', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      externalUrl: 'https://example.com/image.jpg',
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        'Check this out:',
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'https://example.com/image.jpg',
+          inferenceDetail: 'high',
+          _cache: {},
+        },
+      ],
+    });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'input_image',
+            image_url: 'https://example.com/image.jpg',
+            detail: 'high',
+          },
+          { type: 'input_text', text: 'Check this out:' },
+        ],
+      },
+    ]);
+  });
+
+  it('should handle tool calls as top-level function_call items', async () => {
+    const ctx = ChatContext.empty();
+
+    const msg = ctx.addMessage({ role: 'assistant', content: 'Let me help you.' });
+    const toolCall = FunctionCall.create({
+      id: msg.id + '/tool_1',
+      callId: 'call_123',
+      name: 'get_weather',
+      args: '{"location": "Paris"}',
+    });
+    const toolOutput = FunctionCallOutput.create({
+      callId: 'call_123',
+      output: '{"temperature": 20}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall, toolOutput]);
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      { role: 'assistant', content: 'Let me help you.' },
+      {
+        type: 'function_call',
+        call_id: 'call_123',
+        name: 'get_weather',
+        arguments: '{"location": "Paris"}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_123',
+        output: '{"temperature": 20}',
+      },
+    ]);
+  });
+
+  it('should handle tool calls without an accompanying message', async () => {
+    const ctx = ChatContext.empty();
+
+    const toolCall = new FunctionCall({
+      id: 'func_1',
+      callId: 'call_456',
+      name: 'calculate',
+      args: '{"a": 5, "b": 3}',
+    });
+    const toolOutput = new FunctionCallOutput({
+      callId: 'call_456',
+      output: '{"result": 8}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall, toolOutput]);
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      {
+        type: 'function_call',
+        call_id: 'call_456',
+        name: 'calculate',
+        arguments: '{"a": 5, "b": 3}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_456',
+        output: '{"result": 8}',
+      },
+    ]);
+  });
+
+  it('should handle multiple tool calls as separate function_call items', async () => {
+    const ctx = ChatContext.empty();
+
+    const msg = ctx.addMessage({ role: 'assistant', content: "I'll check both." });
+    const toolCall1 = new FunctionCall({
+      id: msg.id + '/tool_1',
+      callId: 'call_1',
+      name: 'get_weather',
+      args: '{"location": "NYC"}',
+    });
+    const toolCall2 = new FunctionCall({
+      id: msg.id + '/tool_2',
+      callId: 'call_2',
+      name: 'get_weather',
+      args: '{"location": "LA"}',
+    });
+    const toolOutput1 = new FunctionCallOutput({
+      callId: 'call_1',
+      output: '{"temperature": 65}',
+      isError: false,
+    });
+    const toolOutput2 = new FunctionCallOutput({
+      callId: 'call_2',
+      output: '{"temperature": 78}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall1, toolCall2, toolOutput1, toolOutput2]);
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      { role: 'assistant', content: "I'll check both." },
+      {
+        type: 'function_call',
+        call_id: 'call_1',
+        name: 'get_weather',
+        arguments: '{"location": "NYC"}',
+      },
+      {
+        type: 'function_call',
+        call_id: 'call_2',
+        name: 'get_weather',
+        arguments: '{"location": "LA"}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_1',
+        output: '{"temperature": 65}',
+      },
+      {
+        type: 'function_call_output',
+        call_id: 'call_2',
+        output: '{"temperature": 78}',
+      },
+    ]);
+  });
+
+  it('should skip empty groups', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: 'Hello', createdAt: 1000 });
+
+    const orphanOutput = new FunctionCallOutput({
+      callId: 'orphan_call',
+      output: 'This should be ignored',
+      isError: false,
+      createdAt: 2000,
+    });
+    ctx.insert(orphanOutput);
+
+    ctx.addMessage({ role: 'assistant', content: 'Hi!', createdAt: 3000 });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toHaveLength(2);
+    expect(result).toContainEqual({ role: 'user', content: 'Hello' });
+    expect(result).toContainEqual({ role: 'assistant', content: 'Hi!' });
+  });
+
+  it('should filter out agent handoff items', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const result = await toResponsesChatCtx(ctx);
+
+    expect(result).toEqual([
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there!' },
+    ]);
+  });
+
+  it('should cache serialized images', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      mimeType: 'image/png',
+      base64Data: 'cached-data',
+    });
+
+    const imageContent = {
+      id: 'img1',
+      type: 'image_content' as const,
+      image: 'https://example.com/image.jpg',
+      inferenceDetail: 'high' as const,
+      _cache: {},
+    };
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({ role: 'user', content: [imageContent] });
+
+    await toResponsesChatCtx(ctx);
+    await toResponsesChatCtx(ctx);
+
+    expect(serializeImageMock).toHaveBeenCalledTimes(1);
+    expect(imageContent._cache).toHaveProperty('serialized_image');
+  });
+
+  it('should throw error for unsupported content type', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          type: 'audio_content',
+          frame: [],
+        },
+      ],
+    });
+
+    await expect(toResponsesChatCtx(ctx)).rejects.toThrow(
+      'Unsupported content type: audio_content',
+    );
+  });
+
+  it('should throw error when serialized image has no data', async () => {
+    serializeImageMock.mockResolvedValue({
+      inferenceDetail: 'high',
+      // No base64Data or externalUrl
+    });
+
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        {
+          id: 'img1',
+          type: 'image_content',
+          image: 'invalid-image',
+          inferenceDetail: 'high',
+          _cache: {},
+        },
+      ],
+    });
+
+    await expect(toResponsesChatCtx(ctx)).rejects.toThrow('Serialized image has no data bytes');
+  });
+});

package/src/llm/provider_format/openai.ts

@@ -144,3 +144,106 @@ async function toImageContent(content: ImageContent) {
     },
   };
 }
+
+async function toResponsesImageContent(content: ImageContent) {
+  const cacheKey = 'serialized_image';
+  let serialized: SerializedImage;
+
+  if (content._cache[cacheKey] === undefined) {
+    serialized = await serializeImage(content);
+    content._cache[cacheKey] = serialized;
+  }
+  serialized = content._cache[cacheKey];
+
+  if (serialized.externalUrl) {
+    return {
+      type: 'input_image' as const,
+      image_url: serialized.externalUrl,
+      detail: serialized.inferenceDetail,
+    };
+  }
+
+  if (serialized.base64Data === undefined) {
+    throw new Error('Serialized image has no data bytes');
+  }
+
+  return {
+    type: 'input_image' as const,
+    image_url: `data:${serialized.mimeType};base64,${serialized.base64Data}`,
+    detail: serialized.inferenceDetail,
+  };
+}
+
+export async function toResponsesChatCtx(
+  chatCtx: ChatContext,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  injectDummyUserMessage: boolean = true,
+) {
+  const itemGroups = groupToolCalls(chatCtx);
+  const messages: Record<string, any>[] = []; // eslint-disable-line @typescript-eslint/no-explicit-any
+
+  for (const group of itemGroups) {
+    if (group.isEmpty) continue;
+
+    if (group.message) {
+      messages.push(await toResponsesChatItem(group.message));
+    }
+
+    for (const toolCall of group.toolCalls) {
+      messages.push({
+        type: 'function_call',
+        call_id: toolCall.callId,
+        name: toolCall.name,
+        arguments: toolCall.args,
+      });
+    }
+
+    for (const toolOutput of group.toolOutputs) {
+      messages.push(await toResponsesChatItem(toolOutput));
+    }
+  }
+
+  return messages;
+}
+
+async function toResponsesChatItem(item: ChatItem) {
+  if (item.type === 'message') {
+    const listContent: Record<string, any>[] = []; // eslint-disable-line @typescript-eslint/no-explicit-any
+    let textContent = '';
+
+    for (const content of item.content) {
+      if (typeof content === 'string') {
+        if (textContent) textContent += '\n';
+        textContent += content;
+      } else if (content.type === 'image_content') {
+        listContent.push(await toResponsesImageContent(content));
+      } else {
+        throw new Error(`Unsupported content type: ${content.type}`);
+      }
+    }
+
+    const content =
+      listContent.length == 0
+        ? textContent
+        : textContent.length == 0
+          ? listContent
+          : [...listContent, { type: 'input_text', text: textContent }];
+
+    return { role: item.role, content };
+  } else if (item.type === 'function_call') {
+    return {
+      type: 'function_call',
+      call_id: item.callId,
+      name: item.name,
+      arguments: item.args,
+    };
+  } else if (item.type === 'function_call_output') {
+    return {
+      type: 'function_call_output',
+      call_id: item.callId,
+      output: item.output,
+    };
+  }
+
+  throw new Error(`Unsupported item type: ${item['type']}`);
+}

package/src/llm/provider_format/utils.ts

@@ -56,12 +56,14 @@ class ChatItemGroup {
   }
 
   removeInvalidToolCalls() {
-    if (this.toolCalls.length === this.toolOutputs.length) {
-      return;
-    }
-
     const toolCallIds = new Set(this.toolCalls.map((call) => call.callId));
     const toolOutputIds = new Set(this.toolOutputs.map((output) => output.callId));
+    const sameIds =
+      toolCallIds.size === toolOutputIds.size &&
+      [...toolCallIds].every((id) => toolOutputIds.has(id));
+    if (this.toolCalls.length === this.toolOutputs.length && sameIds) {
+      return;
+    }
 
     // intersection of tool call ids and tool output ids
     const validCallIds = intersection(toolCallIds, toolOutputIds);

package/src/llm/realtime.ts

@@ -48,6 +48,7 @@ export interface RealtimeCapabilities {
   userTranscription: boolean;
   autoToolReplyGeneration: boolean;
   audioOutput: boolean;
+  manualFunctionCalls: boolean;
 }
 
 export interface InputTranscriptionCompleted {

package/src/llm/tool_context.ts

@@ -80,6 +80,13 @@ export class ToolError extends Error {
   }
 }
 
+export const ToolFlag = {
+  NONE: 0,
+  IGNORE_ON_ENTER: 1 << 0,
+} as const;
+
+export type ToolFlag = (typeof ToolFlag)[keyof typeof ToolFlag];
+
 export interface AgentHandoff {
   /**
    * The agent to handoff to.
@@ -178,6 +185,8 @@ export interface FunctionTool<
    */
   execute: ToolExecuteFunction<Parameters, UserData, Result>;
 
+  flags: number;
+
   [FUNCTION_TOOL_SYMBOL]: true;
 }
 
@@ -242,10 +251,12 @@ export function tool<
   description,
   parameters,
   execute,
+  flags,
 }: {
   description: string;
   parameters: Schema;
   execute: ToolExecuteFunction<InferToolInput<Schema>, UserData, Result>;
+  flags?: number;
 }): FunctionTool<InferToolInput<Schema>, UserData, Result>;
 
 /**
@@ -254,10 +265,12 @@ export function tool<
 export function tool<UserData = UnknownUserData, Result = unknown>({
   description,
   execute,
+  flags,
 }: {
   description: string;
   parameters?: never;
   execute: ToolExecuteFunction<Record<string, never>, UserData, Result>;
+  flags?: number;
 }): FunctionTool<Record<string, never>, UserData, Result>;
 
 /**
@@ -295,6 +308,7 @@ export function tool(tool: any): any {
       description: tool.description,
       parameters,
       execute: tool.execute,
+      flags: tool.flags ?? ToolFlag.NONE,
       [TOOL_SYMBOL]: true,
       [FUNCTION_TOOL_SYMBOL]: true,
     };

package/src/log.ts

@@ -44,7 +44,7 @@ export const log = () => {
 export const initializeLogger = ({ pretty, level }: LoggerOptions) => {
   globals[LOGGER_OPTIONS_KEY] = { pretty, level };
   globals[LOGGER_KEY] = pino(
-    { level: level || 'info' },
+    { level: level || 'info', serializers: { error: pino.stdSerializers.err } },
     pretty ? pinoPretty({ colorize: true }) : process.stdout,
   );
 };
@@ -90,5 +90,8 @@ export const enableOtelLogging = () => {
     { stream: new OtelDestination(), level: 'debug' },
   ];
 
-  globals[LOGGER_KEY] = pino({ level: logLevel }, multistream(streams));
+  globals[LOGGER_KEY] = pino(
+    { level: logLevel, serializers: { error: pino.stdSerializers.err } },
+    multistream(streams),
+  );
 };

package/src/stream/deferred_stream.ts

@@ -59,16 +59,17 @@ export class DeferredReadableStream<T> {
       throw new Error('Stream source already set');
     }
 
-    this.sourceReader = source.getReader();
-    this.pump();
+    const sourceReader = source.getReader();
+    this.sourceReader = sourceReader;
+    void this.pump(sourceReader);
   }
 
-  private async pump() {
+  private async pump(sourceReader: ReadableStreamDefaultReader<T>) {
     let sourceError: unknown;
 
     try {
       while (true) {
-        const { done, value } = await this.sourceReader!.read();
+        const { done, value } = await sourceReader.read();
         if (done) break;
         await this.writer.write(value);
       }
@@ -81,7 +82,7 @@ export class DeferredReadableStream<T> {
       // any other error from source will be propagated to the consumer
       if (sourceError) {
         try {
-          this.writer.abort(sourceError);
+          await this.writer.abort(sourceError);
         } catch (e) {
           // ignore if writer is already closed
         }
@@ -118,10 +119,20 @@ export class DeferredReadableStream<T> {
       return;
     }
 
+    const sourceReader = this.sourceReader!;
+    // Clear source first so future setSource() calls can reattach cleanly.
+    this.sourceReader = undefined;
+
     // release lock will make any pending read() throw TypeError
     // which are expected, and we intentionally catch those error
     // using isStreamReaderReleaseError
     // this will unblock any pending read() inside the async for loop
-    this.sourceReader!.releaseLock();
+    try {
+      sourceReader.releaseLock();
+    } catch (e) {
+      if (!isStreamReaderReleaseError(e)) {
+        throw e;
+      }
+    }
   }
 }