@livekit/agents 0.0.0-20260120144724

package/src/llm/zod-utils.ts

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { JSONSchema7 } from 'json-schema';
+import { toStrictJsonSchema } from 'openai/lib/transform';
+import { zodToJsonSchema as zodToJsonSchemaV3 } from 'zod-to-json-schema';
+import type * as z3 from 'zod/v3';
+import * as z4 from 'zod/v4';
+
+/**
+ * Result type from Zod schema parsing.
+ */
+export type ZodParseResult<T = unknown> =
+  | { success: true; data: T }
+  | { success: false; error: unknown };
+
+/**
+ * Type definition for Zod schemas that works with both v3 and v4.
+ * Uses a union type of both Zod v3 and v4 schema types.
+ *
+ * Adapted from Vercel AI SDK's zodSchema function signature.
+ * Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L278-L281
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type ZodSchema = z4.core.$ZodType<any, any> | z3.Schema<any, z3.ZodTypeDef, any>;
+
+/**
+ * Detects if a schema is a Zod v4 schema.
+ * Zod v4 schemas have a `_zod` property that v3 schemas don't have.
+ *
+ * @param schema - The schema to check
+ * @returns True if the schema is a Zod v4 schema
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Required to match Zod v4's type signature
+export function isZod4Schema(schema: ZodSchema): schema is z4.core.$ZodType<any, any> {
+  // https://zod.dev/library-authors?id=how-to-support-zod-3-and-zod-4-simultaneously
+  return '_zod' in schema;
+}
+
+/**
+ * Checks if a value is a Zod schema (either v3 or v4).
+ *
+ * @param value - The value to check
+ * @returns True if the value is a Zod schema
+ */
+export function isZodSchema(value: unknown): value is ZodSchema {
+  if (typeof value !== 'object' || value === null) {
+    return false;
+  }
+
+  // Check for v4 schema (_zod property)
+  if ('_zod' in value) {
+    return true;
+  }
+
+  // Check for v3 schema (_def property with typeName)
+  if ('_def' in value && typeof value._def === 'object' && value._def !== null) {
+    const def = value._def as Record<string, unknown>;
+    if ('typeName' in def) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Checks if a Zod schema is an object schema.
+ *
+ * @param schema - The schema to check
+ * @returns True if the schema is an object schema
+ */
+export function isZodObjectSchema(schema: ZodSchema): boolean {
+  // Need to access internal Zod properties to check schema type
+  const schemaWithInternals = schema as {
+    _def?: { type?: string; typeName?: string };
+    _zod?: { traits?: Set<string> };
+  };
+
+  // Check for v4 schema first
+  if (isZod4Schema(schema)) {
+    // v4 uses _def.type and _zod.traits
+    return (
+      schemaWithInternals._def?.type === 'object' ||
+      schemaWithInternals._zod?.traits?.has('ZodObject') ||
+      false
+    );
+  }
+
+  // v3 uses _def.typeName
+  return schemaWithInternals._def?.typeName === 'ZodObject';
+}
+
+/**
+ * Converts a Zod schema to JSON Schema format.
+ * Handles both Zod v3 and v4 schemas automatically.
+ *
+ * Adapted from Vercel AI SDK's zod3Schema and zod4Schema functions.
+ * Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L237-L269
+ *
+ * @param schema - The Zod schema to convert
+ * @param isOpenai - Whether to use OpenAI-specific formatting (default: true)
+ * @returns A JSON Schema representation of the Zod schema
+ */
+export function zodSchemaToJsonSchema(
+  schema: ZodSchema,
+  isOpenai: boolean = true,
+  strict: boolean = false,
+): JSONSchema7 {
+  let result: JSONSchema7;
+
+  if (isZod4Schema(schema)) {
+    // Zod v4 has native toJSONSchema support
+    // Configuration adapted from Vercel AI SDK to support OpenAPI conversion for Google
+    // Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L255-L258
+    result = z4.toJSONSchema(schema, {
+      target: 'draft-7',
+      io: 'output',
+      reused: 'inline', // Don't use references by default (to support openapi conversion for google)
+    }) as JSONSchema7;
+  } else {
+    // Zod v3 requires the zod-to-json-schema library
+    // Configuration adapted from Vercel AI SDK
+    // $refStrategy: 'none' is equivalent to v4's reused: 'inline'
+    result = zodToJsonSchemaV3(schema, {
+      target: isOpenai ? 'openAi' : 'jsonSchema7',
+      $refStrategy: 'none', // Don't use references by default (to support openapi conversion for google)
+    }) as JSONSchema7;
+  }
+
+  return strict ? (toStrictJsonSchema(result) as JSONSchema7) : result;
+}
+
+/**
+ * Parses a value against a Zod schema.
+ * Handles both Zod v3 and v4 parse APIs automatically.
+ *
+ * @param schema - The Zod schema to parse against
+ * @param value - The value to parse
+ * @returns A promise that resolves to the parse result
+ */
+export async function parseZodSchema<T = unknown>(
+  schema: ZodSchema,
+  value: unknown,
+): Promise<ZodParseResult<T>> {
+  if (isZod4Schema(schema)) {
+    const result = await z4.safeParseAsync(schema, value);
+    return result as ZodParseResult<T>;
+  } else {
+    const result = await schema.safeParseAsync(value);
+    return result as ZodParseResult<T>;
+  }
+}

package/src/log.ts

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { Writable } from 'node:stream';
+import type { DestinationStream, Logger } from 'pino';
+import { multistream, pino } from 'pino';
+import { build as pinoPretty } from 'pino-pretty';
+import { type PinoLogObject, emitToOtel } from './telemetry/pino_otel_transport.js';
+
+/** @internal */
+export type LoggerOptions = {
+  pretty: boolean;
+  level?: string;
+};
+
+/** @internal */
+export let loggerOptions: LoggerOptions;
+
+/** @internal */
+let logger: Logger | undefined = undefined;
+
+/** @internal */
+let otelEnabled = false;
+
+/** @internal */
+export const log = () => {
+  if (!logger) {
+    throw new TypeError('logger not initialized. did you forget to run initializeLogger()?');
+  }
+  return logger;
+};
+
+/** @internal */
+export const initializeLogger = ({ pretty, level }: LoggerOptions) => {
+  loggerOptions = { pretty, level };
+  logger = pino(
+    { level: level || 'info' },
+    pretty ? pinoPretty({ colorize: true }) : process.stdout,
+  );
+};
+
+/**
+ * Custom Pino destination that parses JSON logs and emits to OTEL.
+ * This receives the FULL serialized log including msg, level, time, etc.
+ */
+class OtelDestination extends Writable {
+  _write(chunk: Buffer, _encoding: string, callback: (error?: Error | null) => void): void {
+    try {
+      const line = chunk.toString().trim();
+      if (line) {
+        const logObj = JSON.parse(line) as PinoLogObject;
+        emitToOtel(logObj);
+      }
+    } catch {
+      // Ignore parse errors (e.g., non-JSON lines)
+    }
+    callback();
+  }
+}
+
+/**
+ * Enable OTEL logging by reconfiguring the logger with multistream.
+ * Uses a custom destination that receives full JSON logs (with msg, level, time).
+ *
+ * @internal
+ */
+export const enableOtelLogging = () => {
+  if (otelEnabled || !logger) {
+    console.warn('OTEL logging already enabled or logger not initialized');
+    return;
+  }
+  otelEnabled = true;
+
+  const { pretty, level } = loggerOptions;
+
+  const logLevel = level || 'info';
+  const streams: { stream: DestinationStream; level: string }[] = [
+    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: logLevel },
+    { stream: new OtelDestination(), level: 'debug' },
+  ];
+
+  logger = pino({ level: logLevel }, multistream(streams));
+};

package/src/metrics/base.ts

@@ -0,0 +1,168 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type AgentMetrics =
+  | STTMetrics
+  | LLMMetrics
+  | TTSMetrics
+  | VADMetrics
+  | EOUMetrics
+  | RealtimeModelMetrics;
+
+export type LLMMetrics = {
+  type: 'llm_metrics';
+  label: string;
+  requestId: string;
+  timestamp: number;
+  /** Duration of the request in milliseconds. */
+  durationMs: number;
+  /** Time to first token in milliseconds. */
+  ttftMs: number;
+  cancelled: boolean;
+  completionTokens: number;
+  promptTokens: number;
+  promptCachedTokens: number;
+  totalTokens: number;
+  tokensPerSecond: number;
+  speechId?: string;
+};
+
+export type STTMetrics = {
+  type: 'stt_metrics';
+  label: string;
+  requestId: string;
+  timestamp: number;
+  /**
+   * The request duration in milliseconds, 0.0 if the STT is streaming.
+   */
+  durationMs: number;
+  /**
+   * The duration of the pushed audio in milliseconds.
+   */
+  audioDurationMs: number;
+  /**
+   * Whether the STT is streaming (e.g using websocket).
+   */
+  streamed: boolean;
+};
+
+export type TTSMetrics = {
+  type: 'tts_metrics';
+  label: string;
+  requestId: string;
+  timestamp: number;
+  /** Time to first byte in milliseconds. */
+  ttfbMs: number;
+  /** Total synthesis duration in milliseconds. */
+  durationMs: number;
+  /** Generated audio duration in milliseconds. */
+  audioDurationMs: number;
+  cancelled: boolean;
+  charactersCount: number;
+  streamed: boolean;
+  segmentId?: string;
+  speechId?: string;
+};
+
+export type VADMetrics = {
+  type: 'vad_metrics';
+  label: string;
+  timestamp: number;
+  idleTimeMs: number;
+  inferenceDurationTotalMs: number;
+  inferenceCount: number;
+};
+
+export type EOUMetrics = {
+  type: 'eou_metrics';
+  timestamp: number;
+  /**
+   * Amount of time between the end of speech from VAD and the decision to end the user's turn.
+   * Set to 0.0 if the end of speech was not detected.
+   */
+  endOfUtteranceDelayMs: number;
+  /**
+   * Time taken to obtain the transcript after the end of the user's speech.
+   * Set to 0.0 if the end of speech was not detected.
+   */
+  transcriptionDelayMs: number;
+  /**
+   * Time taken to invoke the user's `Agent.onUserTurnCompleted` callback.
+   */
+  onUserTurnCompletedDelayMs: number;
+  /**
+   * The time the user stopped speaking.
+   */
+  lastSpeakingTimeMs: number;
+  /**
+   * The ID of the speech handle.
+   */
+  speechId?: string;
+};
+
+export type RealtimeModelMetricsCachedTokenDetails = {
+  audioTokens: number;
+  textTokens: number;
+  imageTokens: number;
+};
+
+export type RealtimeModelMetricsInputTokenDetails = {
+  audioTokens: number;
+  textTokens: number;
+  imageTokens: number;
+  cachedTokens: number;
+  cachedTokensDetails?: RealtimeModelMetricsCachedTokenDetails;
+};
+
+export type RealtimeModelMetricsOutputTokenDetails = {
+  textTokens: number;
+  audioTokens: number;
+  imageTokens: number;
+};
+
+export type RealtimeModelMetrics = {
+  type: 'realtime_model_metrics';
+  label: string;
+  requestId: string;
+  /**
+   * The timestamp of the response creation.
+   */
+  timestamp: number;
+  /**
+   * The duration of the response from created to done in milliseconds.
+   */
+  durationMs: number;
+  /**
+   * Time to first audio token in milliseconds. -1 if no audio token was sent.
+   */
+  ttftMs: number;
+  /**
+   * Whether the request was cancelled.
+   */
+  cancelled: boolean;
+  /**
+   * The number of input tokens used in the Response, including text and audio tokens.
+   */
+  inputTokens: number;
+  /**
+   * The number of output tokens sent in the Response, including text and audio tokens.
+   */
+  outputTokens: number;
+  /**
+   * The total number of tokens in the Response.
+   */
+  totalTokens: number;
+  /**
+   * The number of tokens per second.
+   */
+  tokensPerSecond: number;
+  /**
+   * Details about the input tokens used in the Response.
+   */
+  inputTokenDetails: RealtimeModelMetricsInputTokenDetails;
+  /**
+   * Details about the output tokens used in the Response.
+   */
+  outputTokenDetails: RealtimeModelMetricsOutputTokenDetails;
+};

package/src/metrics/index.ts

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type {
+  AgentMetrics,
+  EOUMetrics,
+  LLMMetrics,
+  RealtimeModelMetrics,
+  STTMetrics,
+  TTSMetrics,
+  VADMetrics,
+} from './base.js';
+export { UsageCollector, type UsageSummary } from './usage_collector.js';
+export { logMetrics } from './utils.js';

package/src/metrics/usage_collector.ts

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AgentMetrics } from './base.js';
+
+export interface UsageSummary {
+  llmPromptTokens: number;
+  llmPromptCachedTokens: number;
+  llmCompletionTokens: number;
+  ttsCharactersCount: number;
+  sttAudioDurationMs: number;
+}
+
+export class UsageCollector {
+  private summary: UsageSummary;
+
+  constructor() {
+    this.summary = {
+      llmPromptTokens: 0,
+      llmPromptCachedTokens: 0,
+      llmCompletionTokens: 0,
+      ttsCharactersCount: 0,
+      sttAudioDurationMs: 0,
+    };
+  }
+
+  collect(metrics: AgentMetrics): void {
+    if (metrics.type === 'llm_metrics') {
+      this.summary.llmPromptTokens += metrics.promptTokens;
+      this.summary.llmPromptCachedTokens += metrics.promptCachedTokens;
+      this.summary.llmCompletionTokens += metrics.completionTokens;
+    } else if (metrics.type === 'realtime_model_metrics') {
+      this.summary.llmPromptTokens += metrics.inputTokens;
+      this.summary.llmPromptCachedTokens += metrics.inputTokenDetails.cachedTokens;
+      this.summary.llmCompletionTokens += metrics.outputTokens;
+    } else if (metrics.type === 'tts_metrics') {
+      this.summary.ttsCharactersCount += metrics.charactersCount;
+    } else if (metrics.type === 'stt_metrics') {
+      this.summary.sttAudioDurationMs += metrics.audioDurationMs;
+    }
+  }
+
+  getSummary(): UsageSummary {
+    return { ...this.summary };
+  }
+}

package/src/metrics/utils.ts

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '../log.js';
+import type { AgentMetrics } from './base.js';
+
+function roundTwoDecimals(value: number) {
+  return Math.round(value * 100) / 100;
+}
+
+export const logMetrics = (metrics: AgentMetrics) => {
+  const logger = log();
+  if (metrics.type === 'llm_metrics') {
+    logger
+      .child({
+        ttftMs: roundTwoDecimals(metrics.ttftMs),
+        inputTokens: metrics.promptTokens,
+        promptCachedTokens: metrics.promptCachedTokens,
+        outputTokens: metrics.completionTokens,
+        tokensPerSecond: roundTwoDecimals(metrics.tokensPerSecond),
+      })
+      .info('LLM metrics');
+  } else if (metrics.type === 'realtime_model_metrics') {
+    logger
+      .child({
+        ttftMs: roundTwoDecimals(metrics.ttftMs),
+        input_tokens: metrics.inputTokens,
+        cached_input_tokens: metrics.inputTokenDetails.cachedTokens,
+        output_tokens: metrics.outputTokens,
+        total_tokens: metrics.totalTokens,
+        tokens_per_second: roundTwoDecimals(metrics.tokensPerSecond),
+      })
+      .info('RealtimeModel metrics');
+  } else if (metrics.type === 'tts_metrics') {
+    logger
+      .child({
+        ttfbMs: roundTwoDecimals(metrics.ttfbMs),
+        audioDurationMs: Math.round(metrics.audioDurationMs),
+      })
+      .info('TTS metrics');
+  } else if (metrics.type === 'eou_metrics') {
+    logger
+      .child({
+        endOfUtteranceDelayMs: roundTwoDecimals(metrics.endOfUtteranceDelayMs),
+        transcriptionDelayMs: roundTwoDecimals(metrics.transcriptionDelayMs),
+        onUserTurnCompletedDelayMs: roundTwoDecimals(metrics.onUserTurnCompletedDelayMs),
+      })
+      .info('EOU metrics');
+  } else if (metrics.type === 'vad_metrics') {
+    logger
+      .child({
+        idleTimeMs: Math.round(metrics.idleTimeMs),
+        inferenceDurationTotalMs: Math.round(metrics.inferenceDurationTotalMs),
+        inferenceCount: metrics.inferenceCount,
+      })
+      .info('VAD metrics');
+  } else if (metrics.type === 'stt_metrics') {
+    logger
+      .child({
+        audioDurationMs: Math.round(metrics.audioDurationMs),
+      })
+      .info('STT metrics');
+  }
+};

package/src/plugin.ts

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { EventEmitter } from 'events';
+
+export enum PluginEventTypes {
+  PluginRegistered = 'plugin_registered',
+}
+
+export type PluginEventMap = {
+  [PluginEventTypes.PluginRegistered]: [Plugin];
+};
+
+export abstract class Plugin {
+  static registeredPlugins = [] as Plugin[];
+  static emitter = new EventEmitter<PluginEventMap>();
+
+  #title: string;
+  #version: string;
+  #package: string;
+
+  constructor(opts: { title: string; version: string; package: string }) {
+    this.#title = opts.title;
+    this.#version = opts.version;
+    this.#package = opts.package;
+  }
+
+  static registerPlugin(plugin: Plugin) {
+    Plugin.registeredPlugins.push(plugin);
+    Plugin.emitter.emit(PluginEventTypes.PluginRegistered, plugin);
+  }
+
+  downloadFiles() {}
+
+  get package(): string {
+    return this.#package;
+  }
+
+  get title(): string {
+    return this.#title;
+  }
+
+  get version(): string {
+    return this.#version;
+  }
+}