@livekit/agents 0.0.0-20260120144724

package/src/llm/provider_format/openai.ts

@@ -0,0 +1,146 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js';
+import { type SerializedImage, serializeImage } from '../utils.js';
+import { groupToolCalls } from './utils.js';
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export async function toChatCtx(chatCtx: ChatContext, injectDummyUserMessage: boolean = true) {
+  const itemGroups = groupToolCalls(chatCtx);
+  const messages: Record<string, any>[] = []; // eslint-disable-line @typescript-eslint/no-explicit-any
+
+  for (const group of itemGroups) {
+    if (group.isEmpty) continue;
+
+    const message: Record<string, any> = group.message // eslint-disable-line @typescript-eslint/no-explicit-any
+      ? await toChatItem(group.message)
+      : { role: 'assistant' };
+
+    const toolCalls = group.toolCalls.map((toolCall) => {
+      const tc: Record<string, any> = {
+        type: 'function',
+        id: toolCall.callId,
+        function: { name: toolCall.name, arguments: toolCall.args },
+      };
+
+      // Include provider-specific extra content (e.g., Google thought signatures)
+      const googleExtra = getGoogleExtra(toolCall);
+      if (googleExtra) {
+        tc.extra_content = { google: googleExtra };
+      }
+      return tc;
+    });
+
+    if (toolCalls.length > 0) {
+      message['tool_calls'] = toolCalls;
+    }
+
+    messages.push(message);
+
+    for (const toolOutput of group.toolOutputs) {
+      messages.push(await toChatItem(toolOutput));
+    }
+  }
+
+  return messages;
+}
+
+async function toChatItem(item: ChatItem) {
+  if (item.type === 'message') {
+    const listContent: Record<string, any>[] = []; // eslint-disable-line @typescript-eslint/no-explicit-any
+    let textContent = '';
+
+    for (const content of item.content) {
+      if (typeof content === 'string') {
+        if (textContent) textContent += '\n';
+        textContent += content;
+      } else if (content.type === 'image_content') {
+        listContent.push(await toImageContent(content));
+      } else {
+        throw new Error(`Unsupported content type: ${content.type}`);
+      }
+    }
+
+    const result: Record<string, any> = { role: item.role };
+    if (listContent.length === 0) {
+      result.content = textContent;
+    } else {
+      if (textContent.length > 0) {
+        listContent.push({ type: 'text', text: textContent });
+      }
+      result.content = listContent;
+    }
+
+    return result;
+  } else if (item.type === 'function_call') {
+    const tc: Record<string, any> = {
+      id: item.callId,
+      type: 'function',
+      function: { name: item.name, arguments: item.args },
+    };
+
+    // Include provider-specific extra content (e.g., Google thought signatures)
+    const googleExtra = getGoogleExtra(item);
+    if (googleExtra) {
+      tc.extra_content = { google: googleExtra };
+    }
+
+    return {
+      role: 'assistant',
+      tool_calls: [tc],
+    };
+  } else if (item.type === 'function_call_output') {
+    return {
+      role: 'tool',
+      tool_call_id: item.callId,
+      content: item.output,
+    };
+  }
+  // Skip other item types (e.g., agent_handoff)
+  // These should be filtered by groupToolCalls, but this is a safety net
+  throw new Error(`Unsupported item type: ${item['type']}`);
+}
+
+function getGoogleExtra(
+  item: Partial<{ extra?: Record<string, unknown>; thoughtSignature?: string }>,
+): Record<string, unknown> | undefined {
+  const googleExtra =
+    (item.extra?.google as Record<string, unknown> | undefined) ||
+    (item.thoughtSignature ? { thoughtSignature: item.thoughtSignature } : undefined);
+  return googleExtra;
+}
+
+async function toImageContent(content: ImageContent) {
+  const cacheKey = 'serialized_image'; // TODO: use hash of encoding options if available
+  let serialized: SerializedImage;
+
+  if (content._cache[cacheKey] === undefined) {
+    serialized = await serializeImage(content);
+    content._cache[cacheKey] = serialized;
+  }
+  serialized = content._cache[cacheKey];
+
+  // Convert SerializedImage to OpenAI format
+  if (serialized.externalUrl) {
+    return {
+      type: 'image_url',
+      image_url: {
+        url: serialized.externalUrl,
+        detail: serialized.inferenceDetail,
+      },
+    };
+  }
+
+  if (serialized.base64Data === undefined) {
+    throw new Error('Serialized image has no data bytes');
+  }
+
+  return {
+    type: 'image_url',
+    image_url: {
+      url: `data:${serialized.mimeType};base64,${serialized.base64Data}`,
+      detail: serialized.inferenceDetail,
+    },
+  };
+}

package/src/llm/provider_format/utils.ts

@@ -0,0 +1,187 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../../log.js';
+import type {
+  ChatContext,
+  ChatItem,
+  ChatMessage,
+  FunctionCall,
+  FunctionCallOutput,
+} from '../chat_context.js';
+
+class ChatItemGroup {
+  message?: ChatMessage;
+  toolCalls: FunctionCall[];
+  toolOutputs: FunctionCallOutput[];
+  logger = log();
+
+  constructor(params: {
+    message?: ChatMessage;
+    toolCalls: FunctionCall[];
+    toolOutputs: FunctionCallOutput[];
+  }) {
+    this.message = params.message;
+    this.toolCalls = params.toolCalls;
+    this.toolOutputs = params.toolOutputs;
+  }
+
+  static create(params?: {
+    message?: ChatMessage;
+    toolCalls?: FunctionCall[];
+    toolOutputs?: FunctionCallOutput[];
+  }) {
+    const { message, toolCalls = [], toolOutputs = [] } = params ?? {};
+    return new ChatItemGroup({ message, toolCalls, toolOutputs });
+  }
+
+  get isEmpty() {
+    return (
+      this.message === undefined && this.toolCalls.length === 0 && this.toolOutputs.length === 0
+    );
+  }
+
+  add(item: ChatItem) {
+    if (item.type === 'message') {
+      if (this.message) {
+        throw new Error('only one message is allowed in a group');
+      }
+      this.message = item;
+    } else if (item.type === 'function_call') {
+      this.toolCalls.push(item);
+    } else if (item.type === 'function_call_output') {
+      this.toolOutputs.push(item);
+    }
+    return this;
+  }
+
+  removeInvalidToolCalls() {
+    if (this.toolCalls.length === this.toolOutputs.length) {
+      return;
+    }
+
+    const toolCallIds = new Set(this.toolCalls.map((call) => call.callId));
+    const toolOutputIds = new Set(this.toolOutputs.map((output) => output.callId));
+
+    // intersection of tool call ids and tool output ids
+    const validCallIds = intersection(toolCallIds, toolOutputIds);
+
+    // filter out tool calls that don't have a corresponding tool output
+    this.toolCalls = this.toolCalls.filter((call) => {
+      if (validCallIds.has(call.callId)) return true;
+      this.logger.warn(
+        {
+          callId: call.callId,
+          toolName: call.name,
+        },
+        'function call missing the corresponding function output, ignoring',
+      );
+      return false;
+    });
+
+    // filter out tool outputs that don't have a corresponding tool call
+    this.toolOutputs = this.toolOutputs.filter((output) => {
+      if (validCallIds.has(output.callId)) return true;
+      this.logger.warn(
+        {
+          callId: output.callId,
+          toolName: output.name,
+        },
+        'function output missing the corresponding function call, ignoring',
+      );
+      return false;
+    });
+  }
+
+  flatten() {
+    const items: ChatItem[] = [];
+    if (this.message) items.push(this.message);
+    items.push(...this.toolCalls, ...this.toolOutputs);
+    return items;
+  }
+}
+
+function intersection<T>(set1: Set<T>, set2: Set<T>): Set<T> {
+  return new Set([...set1].filter((item) => set2.has(item)));
+}
+
+/**
+ * Group chat items (messages, function calls, and function outputs)
+ * into coherent groups based on their item IDs and call IDs.
+ *
+ * Each group will contain:
+ * - Zero or one assistant message
+ * - Zero or more function/tool calls
+ * - The corresponding function/tool outputs matched by call_id
+ *
+ * User and system messages are placed in their own individual groups.
+ *
+ * @param chatCtx - The chat context containing all conversation items
+ * @returns A list of ChatItemGroup objects representing the grouped conversation
+ */
+export function groupToolCalls(chatCtx: ChatContext) {
+  const itemGroups: Record<string, ChatItemGroup> = {};
+  const insertionOrder: Record<string, number> = {};
+  const toolOutputs: FunctionCallOutput[] = [];
+  const logger = log();
+
+  let insertionIndex = 0;
+  for (const item of chatCtx.items) {
+    const isAssistantMessage = item.type === 'message' && item.role === 'assistant';
+    const isFunctionCall = item.type === 'function_call';
+    const isFunctionCallOutput = item.type === 'function_call_output';
+
+    if (isAssistantMessage || isFunctionCall) {
+      // only assistant messages and function calls can be grouped
+      // For function calls, use group_id if available (for parallel function calls),
+      // otherwise fall back to id-based grouping for backwards compatibility
+      const groupId =
+        item.type === 'function_call' && item.groupId ? item.groupId : item.id.split('/')[0]!;
+
+      if (itemGroups[groupId] === undefined) {
+        itemGroups[groupId] = ChatItemGroup.create();
+
+        // we use insertion order to sort the groups as they are added to the context
+        // simulating the OrderedDict in python
+        insertionOrder[groupId] = insertionIndex;
+        insertionIndex++;
+      }
+      itemGroups[groupId]!.add(item);
+    } else if (isFunctionCallOutput) {
+      toolOutputs.push(item);
+    } else {
+      itemGroups[item.id] = ChatItemGroup.create().add(item);
+    }
+  }
+
+  // add tool outputs to their corresponding groups
+  const callIdToGroup: Record<string, ChatItemGroup> = {};
+  for (const group of Object.values(itemGroups)) {
+    for (const toolCall of group.toolCalls) {
+      callIdToGroup[toolCall.callId] = group;
+    }
+  }
+
+  for (const toolOutput of toolOutputs) {
+    const group = callIdToGroup[toolOutput.callId];
+    if (group === undefined) {
+      logger.warn(
+        { callId: toolOutput.callId, toolName: toolOutput.name },
+        'function output missing the corresponding function call, ignoring',
+      );
+      continue;
+    }
+    group.add(toolOutput);
+  }
+
+  // validate that each group and remove invalid tool calls and tool outputs
+  for (const group of Object.values(itemGroups)) {
+    group.removeInvalidToolCalls();
+  }
+
+  // sort groups by their item id
+  const orderedGroups = Object.entries(itemGroups)
+    .sort((a, b) => insertionOrder[a[0]]! - insertionOrder[b[0]]!)
+    .map(([, group]) => group);
+  return orderedGroups;
+}

package/src/llm/realtime.ts

@@ -0,0 +1,163 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import type { ReadableStream } from 'node:stream/web';
+import { DeferredReadableStream } from '../stream/deferred_stream.js';
+import { Task } from '../utils.js';
+import type { ChatContext, FunctionCall } from './chat_context.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
+
+export type InputSpeechStartedEvent = object;
+
+export interface InputSpeechStoppedEvent {
+  userTranscriptionEnabled: boolean;
+}
+
+export interface MessageGeneration {
+  messageId: string;
+  textStream: ReadableStream<string>;
+  audioStream: ReadableStream<AudioFrame>;
+  modalities?: Promise<('text' | 'audio')[]>;
+}
+
+export interface GenerationCreatedEvent {
+  messageStream: ReadableStream<MessageGeneration>;
+  functionStream: ReadableStream<FunctionCall>;
+  userInitiated: boolean;
+  /** Response ID for correlating metrics with spans */
+  responseId?: string;
+}
+
+export interface RealtimeModelError {
+  type: 'realtime_model_error';
+  timestamp: number;
+  label: string;
+  error: Error;
+  recoverable: boolean;
+}
+
+export interface RealtimeCapabilities {
+  messageTruncation: boolean;
+  turnDetection: boolean;
+  userTranscription: boolean;
+  autoToolReplyGeneration: boolean;
+  audioOutput: boolean;
+}
+
+export interface InputTranscriptionCompleted {
+  itemId: string;
+  transcript: string;
+  isFinal: boolean;
+}
+
+export interface RealtimeSessionReconnectedEvent {}
+
+export abstract class RealtimeModel {
+  private _capabilities: RealtimeCapabilities;
+
+  constructor(capabilities: RealtimeCapabilities) {
+    this._capabilities = capabilities;
+  }
+
+  get capabilities() {
+    return this._capabilities;
+  }
+
+  /** The model name/identifier used by this realtime model */
+  abstract get model(): string;
+
+  abstract session(): RealtimeSession;
+
+  abstract close(): Promise<void>;
+}
+
+export abstract class RealtimeSession extends EventEmitter {
+  protected _realtimeModel: RealtimeModel;
+  private deferredInputStream = new DeferredReadableStream<AudioFrame>();
+  private _mainTask: Task<void>;
+
+  constructor(realtimeModel: RealtimeModel) {
+    super();
+    this._realtimeModel = realtimeModel;
+    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));
+  }
+
+  get realtimeModel() {
+    return this._realtimeModel;
+  }
+
+  abstract get chatCtx(): ChatContext;
+
+  abstract get tools(): ToolContext;
+
+  abstract updateInstructions(instructions: string): Promise<void>;
+
+  /**
+   * @throws RealtimeError on Timeout
+   */
+  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;
+
+  abstract updateTools(tools: ToolContext): Promise<void>;
+
+  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;
+
+  abstract pushAudio(frame: AudioFrame): void;
+
+  /**
+   * @throws RealtimeError on Timeout
+   */
+  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;
+
+  /**
+   * Commit the input audio buffer to the server
+   */
+  abstract commitAudio(): Promise<void>;
+
+  /**
+   * Clear the input audio buffer to the server
+   */
+  abstract clearAudio(): Promise<void>;
+
+  /**
+   * Cancel the current generation (do nothing if no generation is in progress)
+   */
+  abstract interrupt(): Promise<void>;
+
+  /**
+   * Truncate the message at the given audio end time
+   */
+  abstract truncate(options: {
+    messageId: string;
+    audioEndMs: number;
+    modalities?: ('text' | 'audio')[];
+    audioTranscript?: string;
+  }): Promise<void>;
+
+  async close(): Promise<void> {
+    this._mainTask.cancel();
+  }
+
+  /**
+   * Notifies the model that user activity has started
+   */
+  startUserActivity(): void {
+    return;
+  }
+
+  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {
+    const reader = this.deferredInputStream.stream.getReader();
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done || signal.aborted) {
+        break;
+      }
+      this.pushAudio(value);
+    }
+  }
+
+  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {
+    this.deferredInputStream.setSource(audioStream);
+  }
+}