@livekit/agents 0.7.9 → 1.0.0-next.0

package/src/llm/utils.ts

@@ -0,0 +1,324 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
+import type { JSONSchema7 } from 'json-schema';
+import sharp from 'sharp';
+import { ZodObject } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import type { UnknownUserData } from '../voice/run_context.js';
+import type { ChatContext } from './chat_context.js';
+import {
+  type ChatItem,
+  FunctionCall,
+  FunctionCallOutput,
+  type ImageContent,
+} from './chat_context.js';
+import type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';
+
+export interface SerializedImage {
+  inferenceDetail: 'auto' | 'high' | 'low';
+  mimeType?: string;
+  base64Data?: string;
+  externalUrl?: string;
+}
+
+function getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {
+  switch (type) {
+    case VideoBufferType.RGBA:
+    case VideoBufferType.ABGR:
+    case VideoBufferType.ARGB:
+    case VideoBufferType.BGRA:
+      return 4;
+    case VideoBufferType.RGB24:
+      return 3;
+    default:
+      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion
+      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);
+  }
+}
+
+function ensureRGBCompatible(frame: VideoFrame): VideoFrame {
+  // If the frame is already in an RGB/RGBA-compatible format, return it directly
+  if (
+    frame.type === VideoBufferType.RGBA ||
+    frame.type === VideoBufferType.BGRA ||
+    frame.type === VideoBufferType.ARGB ||
+    frame.type === VideoBufferType.ABGR ||
+    frame.type === VideoBufferType.RGB24
+  ) {
+    return frame;
+  }
+
+  // Otherwise, attempt conversion for other formats (like YUV)
+  try {
+    return frame.convert(VideoBufferType.RGBA);
+  } catch (error) {
+    throw new Error(
+      `Failed to convert format ${frame.type} to RGB: ${error}. ` +
+        `Consider using RGB/RGBA formats or converting on the client side.`,
+    );
+  }
+}
+
+export async function serializeImage(image: ImageContent): Promise<SerializedImage> {
+  if (typeof image.image === 'string') {
+    if (image.image.startsWith('data:')) {
+      const [header, base64Data] = image.image.split(',', 2) as [string, string];
+      const headerParts = header.split(';');
+      const mimeParts = headerParts[0]?.split(':');
+      const headerMime = mimeParts?.[1];
+
+      if (!headerMime) {
+        throw new Error('Invalid data URL format');
+      }
+
+      let mimeType: string;
+      if (image.mimeType && image.mimeType !== headerMime) {
+        console.warn(
+          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,
+        );
+        mimeType = image.mimeType;
+      } else {
+        mimeType = headerMime;
+      }
+
+      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);
+      if (!supportedTypes.has(mimeType)) {
+        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);
+      }
+
+      return {
+        base64Data,
+        mimeType: mimeType,
+        inferenceDetail: image.inferenceDetail,
+      };
+    }
+
+    // External URL
+    return {
+      mimeType: image.mimeType,
+      inferenceDetail: image.inferenceDetail,
+      externalUrl: image.image,
+    };
+  } else if (image.image instanceof VideoFrame) {
+    const frame = ensureRGBCompatible(image.image);
+    const channels = getChannelsFromVideoBufferType(frame.type);
+
+    // Sharp needs to know the format of raw pixel data
+    let encoded = sharp(frame.data, {
+      raw: {
+        width: frame.width,
+        height: frame.height,
+        channels,
+      },
+    });
+
+    if (image.inferenceWidth && image.inferenceHeight) {
+      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);
+    }
+
+    const base64Data = await encoded
+      .png()
+      .toBuffer()
+      .then((buffer) => buffer.toString('base64'));
+
+    return {
+      base64Data,
+      mimeType: 'image/png',
+      inferenceDetail: image.inferenceDetail,
+    };
+  } else {
+    throw new Error('Unsupported image type');
+  }
+}
+
+/** Raw OpenAI-adherent function parameters. */
+export type OpenAIFunctionParameters = {
+  type: 'object';
+  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any
+  required: string[];
+  additionalProperties?: boolean;
+};
+
+// TODO(brian): remove this helper once we have the real RunContext user data
+export const createToolOptions = <UserData extends UnknownUserData>(
+  toolCallId: string,
+  userData: UserData = {} as UserData,
+): ToolOptions<UserData> => {
+  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;
+};
+
+/** @internal */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const oaiParams = (p: ZodObject<any>): OpenAIFunctionParameters => {
+  // TODO(AJS-162): make zod to JSON parsing able to handle z.optional(v.field())
+  const { properties, required, additionalProperties } = zodToJsonSchema(p, {
+    target: 'openAi',
+  }) as OpenAIFunctionParameters;
+
+  return {
+    type: 'object',
+    properties,
+    required,
+    additionalProperties,
+  };
+};
+
+/** @internal */
+export const oaiBuildFunctionInfo = (
+  toolCtx: ToolContext,
+  toolCallId: string,
+  toolName: string,
+  rawArgs: string,
+): FunctionCall => {
+  const tool = toolCtx[toolName];
+  if (!tool) {
+    throw new Error(`AI tool ${toolName} not found`);
+  }
+
+  return FunctionCall.create({
+    callId: toolCallId,
+    name: toolName,
+    args: rawArgs,
+  });
+};
+
+export async function executeToolCall(
+  toolCall: FunctionCall,
+  toolCtx: ToolContext,
+): Promise<FunctionCallOutput> {
+  const tool = toolCtx[toolCall.name]!;
+  let args: object | undefined;
+  let params: object | undefined;
+
+  // Ensure valid JSON
+  try {
+    args = JSON.parse(toolCall.args);
+  } catch (error) {
+    return FunctionCallOutput.create({
+      callId: toolCall.callId,
+      output: `Invalid JSON: ${error}`,
+      isError: true,
+    });
+  }
+
+  // Ensure valid arguments schema
+  try {
+    if (tool.parameters instanceof ZodObject) {
+      params = tool.parameters.parse(args);
+    } else {
+      params = args;
+    }
+  } catch (error) {
+    return FunctionCallOutput.create({
+      callId: toolCall.callId,
+      output: `Arguments parsing failed: ${error}`,
+      isError: true,
+    });
+  }
+
+  try {
+    const result = await tool.execute(params, createToolOptions(toolCall.callId));
+    return FunctionCallOutput.create({
+      callId: toolCall.callId,
+      output: JSON.stringify(result),
+      isError: false,
+    });
+  } catch (error) {
+    return FunctionCallOutput.create({
+      callId: toolCall.callId,
+      output: `Tool execution failed: ${error}`,
+      isError: true,
+    });
+  }
+}
+
+/**
+ * Standard dynamic-programming LCS to get the common subsequence
+ * of IDs (in order) that appear in both old_ids and new_ids.
+ *
+ * @param oldIds - The old list of IDs.
+ * @param newIds - The new list of IDs.
+ * @returns The longest common subsequence of the two lists of IDs.
+ */
+function computeLCS(oldIds: string[], newIds: string[]): string[] {
+  const n = oldIds.length;
+  const m = newIds.length;
+  const dp: number[][] = Array(n + 1)
+    .fill(null)
+    .map(() => Array(m + 1).fill(0));
+
+  // Fill DP table
+  for (let i = 1; i <= n; i++) {
+    for (let j = 1; j <= m; j++) {
+      if (oldIds[i - 1] === newIds[j - 1]) {
+        dp[i]![j] = dp[i - 1]![j - 1]! + 1;
+      } else {
+        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);
+      }
+    }
+  }
+
+  // Backtrack to find the actual LCS sequence
+  const lcsIds: string[] = [];
+  let i = n;
+  let j = m;
+  while (i > 0 && j > 0) {
+    if (oldIds[i - 1] === newIds[j - 1]) {
+      lcsIds.push(oldIds[i - 1]!);
+      i--;
+      j--;
+    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {
+      i--;
+    } else {
+      j--;
+    }
+  }
+
+  return lcsIds.reverse();
+}
+
+interface DiffOps {
+  toRemove: string[];
+  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root
+}
+
+/**
+ * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.
+ *
+ * @param oldCtx - The old chat context.
+ * @param newCtx - The new chat context.
+ * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.
+ */
+export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {
+  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);
+  const newIds = newCtx.items.map((item: ChatItem) => item.id);
+  const lcsIds = new Set(computeLCS(oldIds, newIds));
+
+  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);
+  const toCreate: Array<[string | null, string]> = [];
+
+  let lastIdInSequence: string | null = null;
+  for (const newItem of newCtx.items) {
+    if (lcsIds.has(newItem.id)) {
+      lastIdInSequence = newItem.id;
+    } else {
+      const prevId = lastIdInSequence; // null if root
+      toCreate.push([prevId, newItem.id]);
+      lastIdInSequence = newItem.id;
+    }
+  }
+
+  return {
+    toRemove,
+    toCreate,
+  };
+}
+
+export function toJsonSchema(schema: ToolInputSchema<any>): JSONSchema7 {
+  if (schema instanceof ZodObject) {
+    return oaiParams(schema);
+  }
+  return schema;
+}

package/src/metrics/base.ts

@@ -2,123 +2,155 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-export interface LLMMetrics {
+export type AgentMetrics =
+  | STTMetrics
+  | LLMMetrics
+  | TTSMetrics
+  | VADMetrics
+  | EOUMetrics
+  | RealtimeModelMetrics;
+
+export type LLMMetrics = {
+  type: 'llm_metrics';
+  label: string;
   requestId: string;
   timestamp: number;
-  ttft: number;
   duration: number;
-  label: string;
+  ttft: number;
   cancelled: boolean;
   completionTokens: number;
   promptTokens: number;
+  promptCachedTokens: number;
   totalTokens: number;
   tokensPerSecond: number;
-  error?: Error;
-}
+  speechId?: string;
+};
 
-export interface STTMetrics {
+export type STTMetrics = {
+  type: 'stt_metrics';
+  label: string;
   requestId: string;
   timestamp: number;
+  /**
+   * The request duration in seconds, 0.0 if the STT is streaming.
+   */
   duration: number;
-  label: string;
+  /**
+   * The duration of the pushed audio in seconds.
+   */
   audioDuration: number;
+  /**
+   * Whether the STT is streaming (e.g using websocket).
+   */
   streamed: boolean;
-  error?: Error;
-}
+};
 
-export interface TTSMetrics {
+export type TTSMetrics = {
+  type: 'tts_metrics';
+  label: string;
   requestId: string;
   timestamp: number;
   ttfb: number;
   duration: number;
-  label: string;
   audioDuration: number;
   cancelled: boolean;
   charactersCount: number;
   streamed: boolean;
-  error?: Error;
-}
+  segmentId?: string;
+  speechId?: string;
+};
 
-export interface VADMetrics {
+export type VADMetrics = {
+  type: 'vad_metrics';
+  label: string;
   timestamp: number;
   idleTime: number;
   inferenceDurationTotal: number;
   inferenceCount: number;
-  label: string;
-}
+};
 
-export interface PipelineEOUMetrics {
+export type EOUMetrics = {
+  type: 'eou_metrics';
+  timestamp: number;
   /**
-   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   * Amount of time between the end of speech from VAD and the decision to end the user's turn.
+   * Set to 0.0 if the end of speech was not detected.
    */
-  sequenceId: string;
-  /** Timestamp of when the event was recorded */
-  timestamp: number;
-  /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
   endOfUtteranceDelay: number;
   /**
    * Time taken to obtain the transcript after the end of the user's speech.
-   *
-   * @remarks
-   * May be 0 if the transcript was already available.
+   * Set to 0.0 if the end of speech was not detected.
    */
   transcriptionDelay: number;
-}
-
-export interface PipelineLLMMetrics extends LLMMetrics {
-  /**
-   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
-   */
-  sequenceId: string;
-}
-
-export interface PipelineTTSMetrics extends TTSMetrics {
   /**
-   * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+   * Time taken to invoke the user's `Agent.onUserTurnCompleted` callback.
    */
-  sequenceId: string;
-}
+  onUserTurnCompletedDelay: number;
+  speechId?: string;
+};
 
-export type PipelineSTTMetrics = STTMetrics;
-export type PipelineVADMetrics = VADMetrics;
+export type RealtimeModelMetricsCachedTokenDetails = {
+  audioTokens: number;
+  textTokens: number;
+  imageTokens: number;
+};
 
-export class MultimodalLLMError extends Error {
-  type?: string;
-  reason?: string;
-  code?: string;
-  constructor({
-    type,
-    reason,
-    code,
-    message,
-  }: { type?: string; reason?: string; code?: string; message?: string } = {}) {
-    super(message);
-    this.type = type;
-    this.reason = reason;
-    this.code = code;
-  }
-}
+export type RealtimeModelMetricsInputTokenDetails = {
+  audioTokens: number;
+  textTokens: number;
+  imageTokens: number;
+  cachedTokens: number;
+  cachedTokensDetails?: RealtimeModelMetricsCachedTokenDetails;
+};
 
-export interface MultimodalLLMMetrics extends LLMMetrics {
-  inputTokenDetails: {
-    cachedTokens: number;
-    textTokens: number;
-    audioTokens: number;
-  };
-  outputTokenDetails: {
-    textTokens: number;
-    audioTokens: number;
-  };
-}
+export type RealtimeModelMetricsOutputTokenDetails = {
+  textTokens: number;
+  audioTokens: number;
+  imageTokens: number;
+};
 
-export type AgentMetrics =
-  | STTMetrics
-  | LLMMetrics
-  | TTSMetrics
-  | VADMetrics
-  | PipelineSTTMetrics
-  | PipelineEOUMetrics
-  | PipelineLLMMetrics
-  | PipelineTTSMetrics
-  | PipelineVADMetrics
-  | MultimodalLLMMetrics;
+export type RealtimeModelMetrics = {
+  type: 'realtime_model_metrics';
+  label: string;
+  requestId: string;
+  /**
+   * The timestamp of the response creation.
+   */
+  timestamp: number;
+  /**
+   * The duration of the response from created to done in seconds.
+   */
+  duration: number;
+  /**
+   * Time to first audio token in seconds. -1 if no audio token was sent.
+   */
+  ttft: number;
+  /**
+   * Whether the request was cancelled.
+   */
+  cancelled: boolean;
+  /**
+   * The number of input tokens used in the Response, including text and audio tokens.
+   */
+  inputTokens: number;
+  /**
+   * The number of output tokens sent in the Response, including text and audio tokens.
+   */
+  outputTokens: number;
+  /**
+   * The total number of tokens in the Response.
+   */
+  totalTokens: number;
+  /**
+   * The number of tokens per second.
+   */
+  tokensPerSecond: number;
+  /**
+   * Details about the input tokens used in the Response.
+   */
+  inputTokenDetails: RealtimeModelMetricsInputTokenDetails;
+  /**
+   * Details about the output tokens used in the Response.
+   */
+  outputTokenDetails: RealtimeModelMetricsOutputTokenDetails;
+};

package/src/metrics/index.ts

@@ -4,17 +4,11 @@
 
 export type {
   AgentMetrics,
-  STTMetrics,
   LLMMetrics,
+  RealtimeModelMetrics,
+  STTMetrics,
   TTSMetrics,
   VADMetrics,
-  PipelineSTTMetrics,
-  PipelineEOUMetrics,
-  PipelineLLMMetrics,
-  PipelineTTSMetrics,
-  PipelineVADMetrics,
-  MultimodalLLMMetrics,
 } from './base.js';
-export { MultimodalLLMError } from './base.js';
-export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { UsageCollector, type UsageSummary } from './usage_collector.js';
 export { logMetrics } from './utils.js';

package/src/metrics/usage_collector.ts

@@ -2,39 +2,45 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { AgentMetrics } from './base.js';
-import { isLLMMetrics, isSTTMetrics, isTTSMetrics } from './utils.js';
 
 export interface UsageSummary {
   llmPromptTokens: number;
+  llmPromptCachedTokens: number;
   llmCompletionTokens: number;
   ttsCharactersCount: number;
   sttAudioDuration: number;
 }
 
 export class UsageCollector {
-  #summary: UsageSummary;
+  private summary: UsageSummary;
 
   constructor() {
-    this.#summary = {
+    this.summary = {
       llmPromptTokens: 0,
+      llmPromptCachedTokens: 0,
       llmCompletionTokens: 0,
       ttsCharactersCount: 0,
       sttAudioDuration: 0,
     };
   }
 
-  collect(metrics: AgentMetrics) {
-    if (isLLMMetrics(metrics)) {
-      this.#summary.llmPromptTokens += metrics.promptTokens;
-      this.#summary.llmCompletionTokens += metrics.completionTokens;
-    } else if (isTTSMetrics(metrics)) {
-      this.#summary.ttsCharactersCount += metrics.charactersCount;
-    } else if (isSTTMetrics(metrics)) {
-      this.#summary.sttAudioDuration += metrics.audioDuration;
+  collect(metrics: AgentMetrics): void {
+    if (metrics.type === 'llm_metrics') {
+      this.summary.llmPromptTokens += metrics.promptTokens;
+      this.summary.llmPromptCachedTokens += metrics.promptCachedTokens;
+      this.summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (metrics.type === 'realtime_model_metrics') {
+      this.summary.llmPromptTokens += metrics.inputTokens;
+      this.summary.llmPromptCachedTokens += metrics.inputTokenDetails.cachedTokens;
+      this.summary.llmCompletionTokens += metrics.outputTokens;
+    } else if (metrics.type === 'tts_metrics') {
+      this.summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (metrics.type === 'stt_metrics') {
+      this.summary.sttAudioDuration += metrics.audioDuration;
     }
   }
 
-  get summary(): UsageSummary {
-    return { ...this.#summary };
+  getSummary(): UsageSummary {
+    return { ...this.summary };
   }
 }