@livekit/agents 1.0.17 → 1.0.19

package/src/llm/provider_format/google.test.ts

@@ -4,7 +4,12 @@
 import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { initializeLogger } from '../../log.js';
-import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js';
+import {
+  AgentHandoffItem,
+  ChatContext,
+  FunctionCall,
+  FunctionCallOutput,
+} from '../chat_context.js';
 import { serializeImage } from '../utils.js';
 import { toChatCtx } from './google.js';
 
@@ -769,4 +774,70 @@ describe('Google Provider Format - toChatCtx', () => {
     ]);
     expect(formatData.systemMessages).toBeNull();
   });
+
+  it('should filter out agent handoff items', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+
+    // Insert an agent handoff item
+    const handoff = new AgentHandoffItem({
+      oldAgentId: 'agent_1',
+      newAgentId: 'agent_2',
+    });
+    ctx.insert(handoff);
+
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const [result, formatData] = await toChatCtx(ctx, false);
+
+    // Agent handoff should be filtered out, only messages should remain
+    expect(result).toEqual([
+      {
+        role: 'user',
+        parts: [{ text: 'Hello' }],
+      },
+      {
+        role: 'model',
+        parts: [{ text: 'Hi there!' }],
+      },
+    ]);
+    expect(formatData.systemMessages).toBeNull();
+  });
+
+  it('should handle multiple agent handoffs without errors', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Start' });
+
+    // Multiple handoffs
+    ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' });
+
+    const [result, formatData] = await toChatCtx(ctx, false);
+
+    // All handoffs should be filtered out
+    // Note: Google provider groups consecutive messages by the same role
+    expect(result).toEqual([
+      {
+        role: 'user',
+        parts: [{ text: 'Start' }],
+      },
+      {
+        role: 'model',
+        parts: [
+          { text: 'Response from agent 1' },
+          { text: 'Response from agent 2' },
+          { text: 'Response from agent 3' },
+        ],
+      },
+    ]);
+    expect(formatData.systemMessages).toBeNull();
+  });
 });

package/src/llm/provider_format/google.ts

@@ -12,11 +12,11 @@ export interface GoogleFormatData {
 export async function toChatCtx(
   chatCtx: ChatContext,
   injectDummyUserMessage: boolean = true,
-): Promise<[Record<string, any>[], GoogleFormatData]> {
-  const turns: Record<string, any>[] = [];
+): Promise<[Record<string, unknown>[], GoogleFormatData]> {
+  const turns: Record<string, unknown>[] = [];
   const systemMessages: string[] = [];
   let currentRole: string | null = null;
-  let parts: Record<string, any>[] = [];
+  let parts: Record<string, unknown>[] = [];
 
   // Flatten all grouped tool calls to get individual messages
   const itemGroups = groupToolCalls(chatCtx);
@@ -104,7 +104,7 @@ export async function toChatCtx(
   ];
 }
 
-async function toImagePart(image: ImageContent): Promise<Record<string, any>> {
+async function toImagePart(image: ImageContent): Promise<Record<string, unknown>> {
   const cacheKey = 'serialized_image';
   if (!image._cache[cacheKey]) {
     image._cache[cacheKey] = await serializeImage(image);

package/src/llm/provider_format/openai.test.ts

@@ -4,7 +4,12 @@
 import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { initializeLogger } from '../../log.js';
-import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js';
+import {
+  AgentHandoffItem,
+  ChatContext,
+  FunctionCall,
+  FunctionCallOutput,
+} from '../chat_context.js';
 import { serializeImage } from '../utils.js';
 import { toChatCtx } from './openai.js';
 
@@ -578,4 +583,53 @@ describe('toChatCtx', () => {
       },
     ]);
   });
+
+  it('should filter out agent handoff items', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+
+    // Insert an agent handoff item
+    const handoff = new AgentHandoffItem({
+      oldAgentId: 'agent_1',
+      newAgentId: 'agent_2',
+    });
+    ctx.insert(handoff);
+
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const result = await toChatCtx(ctx);
+
+    // Agent handoff should be filtered out, only messages should remain
+    expect(result).toEqual([
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there!' },
+    ]);
+  });
+
+  it('should handle multiple agent handoffs without errors', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Start' });
+
+    // Multiple handoffs
+    ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' });
+
+    const result = await toChatCtx(ctx);
+
+    // All handoffs should be filtered out
+    expect(result).toEqual([
+      { role: 'user', content: 'Start' },
+      { role: 'assistant', content: 'Response from agent 1' },
+      { role: 'assistant', content: 'Response from agent 2' },
+      { role: 'assistant', content: 'Response from agent 3' },
+    ]);
+  });
 });

package/src/llm/provider_format/openai.ts

@@ -78,9 +78,10 @@ async function toChatItem(item: ChatItem) {
       tool_call_id: item.callId,
       content: item.output,
     };
-  } else {
-    throw new Error(`Unsupported item type: ${item['type']}`);
   }
+  // Skip other item types (e.g., agent_handoff)
+  // These should be filtered by groupToolCalls, but this is a safety net
+  throw new Error(`Unsupported item type: ${item['type']}`);
 }
 
 async function toImageContent(content: ImageContent) {

package/src/llm/realtime.ts

@@ -19,6 +19,7 @@ export interface MessageGeneration {
   messageId: string;
   textStream: ReadableStream<string>;
   audioStream: ReadableStream<AudioFrame>;
+  modalities?: Promise<('text' | 'audio')[]>;
 }
 
 export interface GenerationCreatedEvent {
@@ -40,6 +41,7 @@ export interface RealtimeCapabilities {
   turnDetection: boolean;
   userTranscription: boolean;
   autoToolReplyGeneration: boolean;
+  audioOutput: boolean;
 }
 
 export interface InputTranscriptionCompleted {
@@ -121,7 +123,12 @@ export abstract class RealtimeSession extends EventEmitter {
   /**
    * Truncate the message at the given audio end time
    */
-  abstract truncate(options: { messageId: string; audioEndMs: number }): Promise<void>;
+  abstract truncate(options: {
+    messageId: string;
+    audioEndMs: number;
+    modalities?: ('text' | 'audio')[];
+    audioTranscript?: string;
+  }): Promise<void>;
 
   async close(): Promise<void> {
     this._mainTask.cancel();

package/src/llm/utils.ts

@@ -323,9 +323,14 @@ export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): Di
   };
 }
 
-export function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {
+export function toJsonSchema(
+  schema: ToolInputSchema<any>,
+  isOpenai: boolean = true,
+  strict: boolean = false,
+): JSONSchema7 {
   if (isZodSchema(schema)) {
-    return zodSchemaToJsonSchema(schema, isOpenai);
+    return zodSchemaToJsonSchema(schema, isOpenai, strict);
   }
+
   return schema as JSONSchema7;
 }

package/src/llm/zod-utils.test.ts

@@ -260,6 +260,107 @@ describe('Zod Utils', () => {
         expect(jsonSchema7).toHaveProperty('properties');
       });
     });
+
+    describe('strict parameter', () => {
+      it('should produce strict JSON schema with strict: true', () => {
+        const schema = z4.object({
+          name: z4.string(),
+          age: z4.number(),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle nullable fields in strict mode', () => {
+        const schema = z4.object({
+          required: z4.string(),
+          optional: z4.string().nullable(),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle default values in strict mode', () => {
+        const schema = z4.object({
+          name: z4.string(),
+          role: z4.string().default('user'),
+          active: z4.boolean().default(true),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle nested objects in strict mode', () => {
+        const schema = z4.object({
+          user: z4.object({
+            name: z4.string(),
+            email: z4.string().nullable(),
+          }),
+          metadata: z4.object({
+            created: z4.string(),
+          }),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle arrays in strict mode', () => {
+        const schema = z4.object({
+          tags: z4.array(z4.string()),
+          numbers: z4.array(z4.number()),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle v3 schemas in strict mode', () => {
+        const schema = z3.object({
+          name: z3.string(),
+          age: z3.number().optional(),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should throw error when using .optional() without .nullable() in strict mode', () => {
+        const schema = z4.object({
+          required: z4.string(),
+          optional: z4.string().optional(),
+        });
+
+        expect(() => zodSchemaToJsonSchema(schema, true, true)).toThrow(
+          /uses `.optional\(\)` without `.nullable\(\)` which is not supported by the API/,
+        );
+      });
+
+      it('should throw error for nested .optional() fields in strict mode', () => {
+        const schema = z4.object({
+          user: z4.object({
+            name: z4.string(),
+            email: z4.string().optional(),
+          }),
+        });
+
+        expect(() => zodSchemaToJsonSchema(schema, true, true)).toThrow(
+          /uses `.optional\(\)` without `.nullable\(\)` which is not supported by the API/,
+        );
+      });
+
+      it('should NOT throw error when using .optional() in non-strict mode', () => {
+        const schema = z4.object({
+          required: z4.string(),
+          optional: z4.string().optional(),
+        });
+
+        expect(() => zodSchemaToJsonSchema(schema, true, false)).not.toThrow();
+      });
+    });
   });
 
   describe('parseZodSchema', () => {

package/src/llm/zod-utils.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { JSONSchema7 } from 'json-schema';
+import { toStrictJsonSchema } from 'openai/lib/transform';
 import { zodToJsonSchema as zodToJsonSchemaV3 } from 'zod-to-json-schema';
 import type * as z3 from 'zod/v3';
 import * as z4 from 'zod/v4';
@@ -101,12 +102,18 @@ export function isZodObjectSchema(schema: ZodSchema): boolean {
  * @param isOpenai - Whether to use OpenAI-specific formatting (default: true)
  * @returns A JSON Schema representation of the Zod schema
  */
-export function zodSchemaToJsonSchema(schema: ZodSchema, isOpenai: boolean = true): JSONSchema7 {
+export function zodSchemaToJsonSchema(
+  schema: ZodSchema,
+  isOpenai: boolean = true,
+  strict: boolean = false,
+): JSONSchema7 {
+  let result: JSONSchema7;
+
   if (isZod4Schema(schema)) {
     // Zod v4 has native toJSONSchema support
     // Configuration adapted from Vercel AI SDK to support OpenAPI conversion for Google
     // Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L255-L258
-    return z4.toJSONSchema(schema, {
+    result = z4.toJSONSchema(schema, {
       target: 'draft-7',
       io: 'output',
       reused: 'inline', // Don't use references by default (to support openapi conversion for google)
@@ -115,11 +122,13 @@ export function zodSchemaToJsonSchema(schema: ZodSchema, isOpenai: boolean = tru
     // Zod v3 requires the zod-to-json-schema library
     // Configuration adapted from Vercel AI SDK
     // $refStrategy: 'none' is equivalent to v4's reused: 'inline'
-    return zodToJsonSchemaV3(schema, {
+    result = zodToJsonSchemaV3(schema, {
       target: isOpenai ? 'openAi' : 'jsonSchema7',
       $refStrategy: 'none', // Don't use references by default (to support openapi conversion for google)
     }) as JSONSchema7;
   }
+
+  return strict ? (toStrictJsonSchema(result) as JSONSchema7) : result;
 }
 
 /**

package/src/log.ts

@@ -42,4 +42,5 @@ export const initializeLogger = ({ pretty, level }: LoggerOptions) => {
   if (level) {
     logger.level = level;
   }
+  // TODO(brian): PR4 - Add Pino bridge to OTEL LoggingHandler for structured logging integration
 };

package/src/telemetry/index.ts

@@ -0,0 +1,10 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// TODO(brian): PR4 - Add logging integration exports
+// TODO(brian): PR5 - Add uploadSessionReport export
+
+export * as traceTypes from './trace_types.js';
+export { setTracerProvider, setupCloudTracer, tracer, type StartSpanOptions } from './traces.js';
+export { recordException, recordRealtimeMetrics } from './utils.js';

package/src/telemetry/trace_types.ts

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// LiveKit custom attributes
+export const ATTR_SPEECH_ID = 'lk.speech_id';
+export const ATTR_AGENT_LABEL = 'lk.agent_label';
+export const ATTR_START_TIME = 'lk.start_time';
+export const ATTR_END_TIME = 'lk.end_time';
+export const ATTR_RETRY_COUNT = 'lk.retry_count';
+
+export const ATTR_PARTICIPANT_ID = 'lk.participant_id';
+export const ATTR_PARTICIPANT_IDENTITY = 'lk.participant_identity';
+export const ATTR_PARTICIPANT_KIND = 'lk.participant_kind';
+
+// session start
+export const ATTR_JOB_ID = 'lk.job_id';
+export const ATTR_AGENT_NAME = 'lk.agent_name';
+export const ATTR_ROOM_NAME = 'lk.room_name';
+export const ATTR_SESSION_OPTIONS = 'lk.session_options';
+
+// assistant turn
+export const ATTR_USER_INPUT = 'lk.user_input';
+export const ATTR_INSTRUCTIONS = 'lk.instructions';
+export const ATTR_SPEECH_INTERRUPTED = 'lk.interrupted';
+
+// llm node
+export const ATTR_CHAT_CTX = 'lk.chat_ctx';
+export const ATTR_FUNCTION_TOOLS = 'lk.function_tools';
+export const ATTR_RESPONSE_TEXT = 'lk.response.text';
+export const ATTR_RESPONSE_FUNCTION_CALLS = 'lk.response.function_calls';
+
+// function tool
+export const ATTR_FUNCTION_TOOL_NAME = 'lk.function_tool.name';
+export const ATTR_FUNCTION_TOOL_ARGS = 'lk.function_tool.arguments';
+export const ATTR_FUNCTION_TOOL_IS_ERROR = 'lk.function_tool.is_error';
+export const ATTR_FUNCTION_TOOL_OUTPUT = 'lk.function_tool.output';
+
+// tts node
+export const ATTR_TTS_INPUT_TEXT = 'lk.input_text';
+export const ATTR_TTS_STREAMING = 'lk.tts.streaming';
+export const ATTR_TTS_LABEL = 'lk.tts.label';
+
+// eou detection
+export const ATTR_EOU_PROBABILITY = 'lk.eou.probability';
+export const ATTR_EOU_UNLIKELY_THRESHOLD = 'lk.eou.unlikely_threshold';
+export const ATTR_EOU_DELAY = 'lk.eou.endpointing_delay';
+export const ATTR_EOU_LANGUAGE = 'lk.eou.language';
+export const ATTR_USER_TRANSCRIPT = 'lk.user_transcript';
+export const ATTR_TRANSCRIPT_CONFIDENCE = 'lk.transcript_confidence';
+export const ATTR_TRANSCRIPTION_DELAY = 'lk.transcription_delay';
+export const ATTR_END_OF_TURN_DELAY = 'lk.end_of_turn_delay';
+
+// metrics
+export const ATTR_LLM_METRICS = 'lk.llm_metrics';
+export const ATTR_TTS_METRICS = 'lk.tts_metrics';
+export const ATTR_REALTIME_MODEL_METRICS = 'lk.realtime_model_metrics';
+
+// OpenTelemetry GenAI attributes
+// OpenTelemetry specification: https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/
+export const ATTR_GEN_AI_OPERATION_NAME = 'gen_ai.operation.name';
+export const ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model';
+export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens';
+export const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens';
+
+// Unofficial OpenTelemetry GenAI attributes, recognized by LangFuse
+// https://langfuse.com/integrations/native/opentelemetry#usage
+// but not yet in the official OpenTelemetry specification.
+export const ATTR_GEN_AI_USAGE_INPUT_TEXT_TOKENS = 'gen_ai.usage.input_text_tokens';
+export const ATTR_GEN_AI_USAGE_INPUT_AUDIO_TOKENS = 'gen_ai.usage.input_audio_tokens';
+export const ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS = 'gen_ai.usage.input_cached_tokens';
+export const ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS = 'gen_ai.usage.output_text_tokens';
+export const ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS = 'gen_ai.usage.output_audio_tokens';
+
+// OpenTelemetry GenAI event names (for structured logging)
+export const EVENT_GEN_AI_SYSTEM_MESSAGE = 'gen_ai.system.message';
+export const EVENT_GEN_AI_USER_MESSAGE = 'gen_ai.user.message';
+export const EVENT_GEN_AI_ASSISTANT_MESSAGE = 'gen_ai.assistant.message';
+export const EVENT_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message';
+export const EVENT_GEN_AI_CHOICE = 'gen_ai.choice';
+
+// Exception attributes
+export const ATTR_EXCEPTION_TRACE = 'exception.stacktrace';
+export const ATTR_EXCEPTION_TYPE = 'exception.type';
+export const ATTR_EXCEPTION_MESSAGE = 'exception.message';
+
+// Platform-specific attributes
+export const ATTR_LANGFUSE_COMPLETION_START_TIME = 'langfuse.observation.completion_start_time';