@livekit/agents 0.7.8 → 1.0.0-next.0

package/src/llm/provider_format/utils.ts

@@ -0,0 +1,183 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../../log.js';
+import type {
+  ChatContext,
+  ChatItem,
+  ChatMessage,
+  FunctionCall,
+  FunctionCallOutput,
+} from '../chat_context.js';
+
+class ChatItemGroup {
+  message?: ChatMessage;
+  toolCalls: FunctionCall[];
+  toolOutputs: FunctionCallOutput[];
+  logger = log();
+
+  constructor(params: {
+    message?: ChatMessage;
+    toolCalls: FunctionCall[];
+    toolOutputs: FunctionCallOutput[];
+  }) {
+    this.message = params.message;
+    this.toolCalls = params.toolCalls;
+    this.toolOutputs = params.toolOutputs;
+  }
+
+  static create(params?: {
+    message?: ChatMessage;
+    toolCalls?: FunctionCall[];
+    toolOutputs?: FunctionCallOutput[];
+  }) {
+    const { message, toolCalls = [], toolOutputs = [] } = params ?? {};
+    return new ChatItemGroup({ message, toolCalls, toolOutputs });
+  }
+
+  get isEmpty() {
+    return (
+      this.message === undefined && this.toolCalls.length === 0 && this.toolOutputs.length === 0
+    );
+  }
+
+  add(item: ChatItem) {
+    if (item.type === 'message') {
+      if (this.message) {
+        throw new Error('only one message is allowed in a group');
+      }
+      this.message = item;
+    } else if (item.type === 'function_call') {
+      this.toolCalls.push(item);
+    } else if (item.type === 'function_call_output') {
+      this.toolOutputs.push(item);
+    }
+    return this;
+  }
+
+  removeInvalidToolCalls() {
+    if (this.toolCalls.length === this.toolOutputs.length) {
+      return;
+    }
+
+    const toolCallIds = new Set(this.toolCalls.map((call) => call.callId));
+    const toolOutputIds = new Set(this.toolOutputs.map((output) => output.callId));
+
+    // intersection of tool call ids and tool output ids
+    const validCallIds = intersection(toolCallIds, toolOutputIds);
+
+    // filter out tool calls that don't have a corresponding tool output
+    this.toolCalls = this.toolCalls.filter((call) => {
+      if (validCallIds.has(call.callId)) return true;
+      this.logger.warn(
+        {
+          callId: call.callId,
+          toolName: call.name,
+        },
+        'function call missing the corresponding function output, ignoring',
+      );
+      return false;
+    });
+
+    // filter out tool outputs that don't have a corresponding tool call
+    this.toolOutputs = this.toolOutputs.filter((output) => {
+      if (validCallIds.has(output.callId)) return true;
+      this.logger.warn(
+        {
+          callId: output.callId,
+          toolName: output.name,
+        },
+        'function output missing the corresponding function call, ignoring',
+      );
+      return false;
+    });
+  }
+
+  flatten() {
+    const items: ChatItem[] = [];
+    if (this.message) items.push(this.message);
+    items.push(...this.toolCalls, ...this.toolOutputs);
+    return items;
+  }
+}
+
+function intersection<T>(set1: Set<T>, set2: Set<T>): Set<T> {
+  return new Set([...set1].filter((item) => set2.has(item)));
+}
+
+/**
+ * Group chat items (messages, function calls, and function outputs)
+ * into coherent groups based on their item IDs and call IDs.
+ *
+ * Each group will contain:
+ * - Zero or one assistant message
+ * - Zero or more function/tool calls
+ * - The corresponding function/tool outputs matched by call_id
+ *
+ * User and system messages are placed in their own individual groups.
+ *
+ * @param chatCtx - The chat context containing all conversation items
+ * @returns A list of ChatItemGroup objects representing the grouped conversation
+ */
+export function groupToolCalls(chatCtx: ChatContext) {
+  const itemGroups: Record<string, ChatItemGroup> = {};
+  const insertionOrder: Record<string, number> = {};
+  const toolOutputs: FunctionCallOutput[] = [];
+  const logger = log();
+
+  let insertionIndex = 0;
+  for (const item of chatCtx.items) {
+    const isAssistantMessage = item.type === 'message' && item.role === 'assistant';
+    const isFunctionCall = item.type === 'function_call';
+    const isFunctionCallOutput = item.type === 'function_call_output';
+
+    if (isAssistantMessage || isFunctionCall) {
+      // only assistant messages and function calls can be grouped
+      const groupId = item.id.split('/')[0]!;
+      if (itemGroups[groupId] === undefined) {
+        itemGroups[groupId] = ChatItemGroup.create();
+
+        // we use insertion order to sort the groups as they are added to the context
+        // simulating the OrderedDict in python
+        insertionOrder[groupId] = insertionIndex;
+        insertionIndex++;
+      }
+      itemGroups[groupId]!.add(item);
+    } else if (isFunctionCallOutput) {
+      toolOutputs.push(item);
+    } else {
+      itemGroups[item.id] = ChatItemGroup.create().add(item);
+    }
+  }
+
+  // add tool outputs to their corresponding groups
+  const callIdToGroup: Record<string, ChatItemGroup> = {};
+  for (const group of Object.values(itemGroups)) {
+    for (const toolCall of group.toolCalls) {
+      callIdToGroup[toolCall.callId] = group;
+    }
+  }
+
+  for (const toolOutput of toolOutputs) {
+    const group = callIdToGroup[toolOutput.callId];
+    if (group === undefined) {
+      logger.warn(
+        { callId: toolOutput.callId, toolName: toolOutput.name },
+        'function output missing the corresponding function call, ignoring',
+      );
+      continue;
+    }
+    group.add(toolOutput);
+  }
+
+  // validate that each group and remove invalid tool calls and tool outputs
+  for (const group of Object.values(itemGroups)) {
+    group.removeInvalidToolCalls();
+  }
+
+  // sort groups by their item id
+  const orderedGroups = Object.entries(itemGroups)
+    .sort((a, b) => insertionOrder[a[0]]! - insertionOrder[b[0]]!)
+    .map(([, group]) => group);
+  return orderedGroups;
+}

package/src/llm/realtime.ts

@@ -0,0 +1,151 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import type { ReadableStream } from 'node:stream/web';
+import { DeferredReadableStream } from '../stream/deferred_stream.js';
+import { Task } from '../utils.js';
+import type { ChatContext, FunctionCall } from './chat_context.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
+
+export type InputSpeechStartedEvent = object;
+
+export interface InputSpeechStoppedEvent {
+  userTranscriptionEnabled: boolean;
+}
+
+export interface MessageGeneration {
+  messageId: string;
+  textStream: ReadableStream<string>;
+  audioStream: ReadableStream<AudioFrame>;
+}
+
+export interface GenerationCreatedEvent {
+  messageStream: ReadableStream<MessageGeneration>;
+  functionStream: ReadableStream<FunctionCall>;
+  userInitiated: boolean;
+}
+
+export interface RealtimeModelError {
+  type: 'realtime_model_error';
+  timestamp: number;
+  label: string;
+  error: Error;
+  recoverable: boolean;
+}
+
+export interface RealtimeCapabilities {
+  messageTruncation: boolean;
+  turnDetection: boolean;
+  userTranscription: boolean;
+  autoToolReplyGeneration: boolean;
+}
+
+export interface InputTranscriptionCompleted {
+  itemId: string;
+  transcript: string;
+  isFinal: boolean;
+}
+
+export interface RealtimeSessionReconnectedEvent {}
+
+export abstract class RealtimeModel {
+  private _capabilities: RealtimeCapabilities;
+
+  constructor(capabilities: RealtimeCapabilities) {
+    this._capabilities = capabilities;
+  }
+
+  get capabilities() {
+    return this._capabilities;
+  }
+
+  abstract session(): RealtimeSession;
+
+  abstract close(): Promise<void>;
+}
+
+export abstract class RealtimeSession extends EventEmitter {
+  protected _realtimeModel: RealtimeModel;
+  private deferredInputStream = new DeferredReadableStream<AudioFrame>();
+  private _mainTask: Task<void>;
+
+  constructor(realtimeModel: RealtimeModel) {
+    super();
+    this._realtimeModel = realtimeModel;
+    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));
+  }
+
+  get realtimeModel() {
+    return this._realtimeModel;
+  }
+
+  abstract get chatCtx(): ChatContext;
+
+  abstract get tools(): ToolContext;
+
+  abstract updateInstructions(instructions: string): Promise<void>;
+
+  /**
+   * @throws RealtimeError on Timeout
+   */
+  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;
+
+  abstract updateTools(tools: ToolContext): Promise<void>;
+
+  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;
+
+  abstract pushAudio(frame: AudioFrame): void;
+
+  /**
+   * @throws RealtimeError on Timeout
+   */
+  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;
+
+  /**
+   * Commit the input audio buffer to the server
+   */
+  abstract commitAudio(): Promise<void>;
+
+  /**
+   * Clear the input audio buffer to the server
+   */
+  abstract clearAudio(): Promise<void>;
+
+  /**
+   * Cancel the current generation (do nothing if no generation is in progress)
+   */
+  abstract interrupt(): Promise<void>;
+
+  /**
+   * Truncate the message at the given audio end time
+   */
+  abstract truncate(options: { messageId: string; audioEndMs: number }): Promise<void>;
+
+  async close(): Promise<void> {
+    this._mainTask.cancel();
+  }
+
+  /**
+   * Notifies the model that user activity has started
+   */
+  startUserActivity(): void {
+    return;
+  }
+
+  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {
+    const reader = this.deferredInputStream.stream.getReader();
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done || signal.aborted) {
+        break;
+      }
+      this.pushAudio(value);
+    }
+  }
+
+  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {
+    this.deferredInputStream.setSource(audioStream);
+  }
+}

package/src/llm/remote_chat_context.test.ts

@@ -0,0 +1,290 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { beforeEach, describe, expect, it } from 'vitest';
+import { ChatMessage } from './chat_context.js';
+import { RemoteChatContext } from './remote_chat_context.js';
+
+function createMessage(id: string, content: string): ChatMessage {
+  return new ChatMessage({ id, role: 'user', content });
+}
+
+describe('RemoteChatContext', () => {
+  let context: RemoteChatContext;
+
+  beforeEach(() => {
+    context = new RemoteChatContext();
+  });
+
+  describe('empty context', () => {
+    it('should return empty ChatContext', () => {
+      const chatCtx = context.toChatCtx();
+      expect(chatCtx.items).toHaveLength(0);
+    });
+
+    it('should return null for non-existent item', () => {
+      expect(context.get('nonexistent')).toBeNull();
+    });
+
+    it('should throw error when deleting non-existent item', () => {
+      expect(() => context.delete('nonexistent')).toThrow('Item with ID nonexistent not found');
+    });
+  });
+
+  describe('single item operations', () => {
+    it('should insert single item at head', () => {
+      const msg = createMessage('msg1', 'Hello');
+      context.insert(undefined, msg);
+
+      expect(context.get('msg1')).toBeDefined();
+      expect(context.get('msg1')!.item).toBe(msg);
+      expect(context.toChatCtx().items).toEqual([msg]);
+    });
+
+    it('should delete single item', () => {
+      const msg = createMessage('msg1', 'Hello');
+      context.insert(undefined, msg);
+      context.delete('msg1');
+
+      expect(context.get('msg1')).toBeNull();
+      expect(context.toChatCtx().items).toHaveLength(0);
+    });
+  });
+
+  describe('multiple item operations', () => {
+    it('should insert multiple items at head', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+
+      context.insert(undefined, msg1);
+      context.insert(undefined, msg2);
+      context.insert(undefined, msg3);
+
+      expect(context.toChatCtx().items).toEqual([msg3, msg2, msg1]);
+    });
+
+    it('should insert items after specific nodes', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg1', msg3);
+
+      expect(context.toChatCtx().items).toEqual([msg1, msg3, msg2]);
+    });
+
+    it('should insert at tail', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg2', msg3);
+
+      expect(context.toChatCtx().items).toEqual([msg1, msg2, msg3]);
+    });
+  });
+
+  describe('deletion edge cases', () => {
+    it('should delete head node from multi-item list', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg2', msg3);
+
+      context.delete('msg1');
+      expect(context.toChatCtx().items).toEqual([msg2, msg3]);
+    });
+
+    it('should delete tail node from multi-item list', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg2', msg3);
+
+      context.delete('msg3');
+      expect(context.toChatCtx().items).toEqual([msg1, msg2]);
+    });
+
+    it('should delete middle node from multi-item list', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg2', msg3);
+
+      context.delete('msg2');
+      expect(context.toChatCtx().items).toEqual([msg1, msg3]);
+    });
+
+    it('should handle multiple deletions', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+      const msg4 = createMessage('msg4', 'Fourth');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg2', msg3);
+      context.insert('msg3', msg4);
+
+      context.delete('msg2');
+      context.delete('msg4');
+      expect(context.toChatCtx().items).toEqual([msg1, msg3]);
+
+      context.delete('msg1');
+      expect(context.toChatCtx().items).toEqual([msg3]);
+
+      context.delete('msg3');
+      expect(context.toChatCtx().items).toHaveLength(0);
+    });
+  });
+
+  describe('error conditions', () => {
+    it('should throw error when inserting duplicate ID', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg1', 'Duplicate');
+
+      context.insert(undefined, msg1);
+      expect(() => context.insert(undefined, msg2)).toThrow('Item with ID msg1 already exists.');
+    });
+
+    it('should throw error when inserting after non-existent ID', () => {
+      const msg = createMessage('msg1', 'Hello');
+      expect(() => context.insert('nonexistent', msg)).toThrow(
+        'previousItemId nonexistent not found',
+      );
+    });
+
+    it('should throw error when deleting non-existent ID', () => {
+      expect(() => context.delete('nonexistent')).toThrow('Item with ID nonexistent not found');
+    });
+  });
+
+  describe('complex scenarios', () => {
+    it('should handle interleaved inserts and deletes', () => {
+      const msg1 = createMessage('msg1', 'A');
+      const msg2 = createMessage('msg2', 'B');
+      const msg3 = createMessage('msg3', 'C');
+      const msg4 = createMessage('msg4', 'D');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.delete('msg1');
+      context.insert('msg2', msg3);
+      context.insert(undefined, msg4);
+
+      expect(context.toChatCtx().items).toEqual([msg4, msg2, msg3]);
+    });
+
+    it('should maintain correct pointers after complex operations', () => {
+      const msg1 = createMessage('msg1', 'A');
+      const msg2 = createMessage('msg2', 'B');
+      const msg3 = createMessage('msg3', 'C');
+      const msg4 = createMessage('msg4', 'D');
+      const msg5 = createMessage('msg5', 'E');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg2', msg3);
+      context.insert('msg1', msg4);
+      context.insert('msg4', msg5);
+
+      expect(context.toChatCtx().items).toEqual([msg1, msg4, msg5, msg2, msg3]);
+
+      context.delete('msg4');
+      expect(context.toChatCtx().items).toEqual([msg1, msg5, msg2, msg3]);
+
+      context.delete('msg1');
+      expect(context.toChatCtx().items).toEqual([msg5, msg2, msg3]);
+
+      context.delete('msg2');
+      expect(context.toChatCtx().items).toEqual([msg5, msg3]);
+    });
+
+    it('should handle rebuilding from scratch', () => {
+      const messages = Array.from({ length: 10 }, (_, i) =>
+        createMessage(`msg${i}`, `Content ${i}`),
+      );
+
+      for (const msg of messages) {
+        context.insert(undefined, msg);
+      }
+
+      expect(context.toChatCtx().items).toEqual([...messages].reverse());
+
+      for (let i = 0; i < 5; i++) {
+        context.delete(`msg${i}`);
+      }
+
+      const remaining = [...messages]
+        .reverse()
+        .filter((msg) => !['msg0', 'msg1', 'msg2', 'msg3', 'msg4'].includes(msg.id));
+      expect(context.toChatCtx().items).toEqual(remaining);
+    });
+  });
+
+  describe('get method', () => {
+    it('should return correct item for existing ID', () => {
+      const msg1 = createMessage('msg1', 'Hello');
+      const msg2 = createMessage('msg2', 'World');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+
+      const retrieved = context.get('msg2');
+      expect(retrieved).toBeDefined();
+      expect(retrieved!.item).toBe(msg2);
+    });
+
+    it('should return null for non-existent ID', () => {
+      const msg = createMessage('msg1', 'Hello');
+      context.insert(undefined, msg);
+
+      expect(context.get('nonexistent')).toBeNull();
+    });
+  });
+
+  describe('toChatCtx method', () => {
+    it('should preserve order in ChatContext', () => {
+      const msg1 = createMessage('msg1', 'First');
+      const msg2 = createMessage('msg2', 'Second');
+      const msg3 = createMessage('msg3', 'Third');
+
+      context.insert(undefined, msg1);
+      context.insert('msg1', msg2);
+      context.insert('msg2', msg3);
+
+      const chatCtx = context.toChatCtx();
+      expect(chatCtx.items).toEqual([msg1, msg2, msg3]);
+    });
+
+    it('should work with empty context', () => {
+      const chatCtx = context.toChatCtx();
+      expect(chatCtx.items).toHaveLength(0);
+    });
+
+    it('should create new ChatContext instance', () => {
+      const msg = createMessage('msg1', 'Hello');
+      context.insert(undefined, msg);
+
+      const chatCtx1 = context.toChatCtx();
+      const chatCtx2 = context.toChatCtx();
+
+      expect(chatCtx1).not.toBe(chatCtx2);
+      expect(chatCtx1.items).toEqual(chatCtx2.items);
+    });
+  });
+});