@livekit/agents 1.0.16 → 1.0.18

package/src/llm/__snapshots__/zod-utils.test.ts.snap

@@ -339,3 +339,221 @@ exports[`Zod Utils > zodSchemaToJsonSchema > Zod v4 schemas > should handle v4 s
   "type": "object",
 }
 `;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle arrays in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "numbers": {
+      "items": {
+        "type": "number",
+      },
+      "type": "array",
+    },
+    "tags": {
+      "items": {
+        "type": "string",
+      },
+      "type": "array",
+    },
+  },
+  "required": [
+    "tags",
+    "numbers",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle default values in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "active": {
+      "default": true,
+      "type": "boolean",
+    },
+    "name": {
+      "type": "string",
+    },
+    "role": {
+      "default": "user",
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "role",
+    "active",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle nested objects in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "metadata": {
+      "additionalProperties": false,
+      "properties": {
+        "created": {
+          "type": "string",
+        },
+      },
+      "required": [
+        "created",
+      ],
+      "type": "object",
+    },
+    "user": {
+      "additionalProperties": false,
+      "properties": {
+        "email": {
+          "anyOf": [
+            {
+              "type": "string",
+            },
+            {
+              "type": "null",
+            },
+          ],
+        },
+        "name": {
+          "type": "string",
+        },
+      },
+      "required": [
+        "name",
+        "email",
+      ],
+      "type": "object",
+    },
+  },
+  "required": [
+    "user",
+    "metadata",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle nullable fields in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "optional": {
+      "anyOf": [
+        {
+          "type": "string",
+        },
+        {
+          "type": "null",
+        },
+      ],
+    },
+    "required": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "required",
+    "optional",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle optional fields in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "optional": {
+      "anyOf": [
+        {
+          "type": "string",
+        },
+        {
+          "type": "null",
+        },
+      ],
+    },
+    "required": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "required",
+    "optional",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle v3 schemas in strict mode 1`] = `
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "age": {
+      "type": [
+        "number",
+        "null",
+      ],
+    },
+    "name": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "age",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should produce standard JSON schema with strict: false 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "age": {
+      "type": "number",
+    },
+    "name": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "age",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should produce strict JSON schema with strict: true 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "age": {
+      "type": "number",
+    },
+    "name": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "age",
+  ],
+  "type": "object",
+}
+`;

package/src/llm/llm.ts

@@ -78,7 +78,7 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCal
     connOptions?: APIConnectOptions;
     parallelToolCalls?: boolean;
     toolChoice?: ToolChoice;
-    extraKwargs?: Record<string, any>;
+    extraKwargs?: Record<string, unknown>;
   }): LLMStream;
 
   /**

package/src/llm/provider_format/google.ts

@@ -12,11 +12,11 @@ export interface GoogleFormatData {
 export async function toChatCtx(
   chatCtx: ChatContext,
   injectDummyUserMessage: boolean = true,
-): Promise<[Record<string, any>[], GoogleFormatData]> {
-  const turns: Record<string, any>[] = [];
+): Promise<[Record<string, unknown>[], GoogleFormatData]> {
+  const turns: Record<string, unknown>[] = [];
   const systemMessages: string[] = [];
   let currentRole: string | null = null;
-  let parts: Record<string, any>[] = [];
+  let parts: Record<string, unknown>[] = [];
 
   // Flatten all grouped tool calls to get individual messages
   const itemGroups = groupToolCalls(chatCtx);
@@ -104,7 +104,7 @@ export async function toChatCtx(
   ];
 }
 
-async function toImagePart(image: ImageContent): Promise<Record<string, any>> {
+async function toImagePart(image: ImageContent): Promise<Record<string, unknown>> {
   const cacheKey = 'serialized_image';
   if (!image._cache[cacheKey]) {
     image._cache[cacheKey] = await serializeImage(image);

package/src/llm/realtime.ts

@@ -19,6 +19,7 @@ export interface MessageGeneration {
   messageId: string;
   textStream: ReadableStream<string>;
   audioStream: ReadableStream<AudioFrame>;
+  modalities?: Promise<('text' | 'audio')[]>;
 }
 
 export interface GenerationCreatedEvent {
@@ -40,6 +41,7 @@ export interface RealtimeCapabilities {
   turnDetection: boolean;
   userTranscription: boolean;
   autoToolReplyGeneration: boolean;
+  audioOutput: boolean;
 }
 
 export interface InputTranscriptionCompleted {
@@ -121,7 +123,12 @@ export abstract class RealtimeSession extends EventEmitter {
   /**
    * Truncate the message at the given audio end time
    */
-  abstract truncate(options: { messageId: string; audioEndMs: number }): Promise<void>;
+  abstract truncate(options: {
+    messageId: string;
+    audioEndMs: number;
+    modalities?: ('text' | 'audio')[];
+    audioTranscript?: string;
+  }): Promise<void>;
 
   async close(): Promise<void> {
     this._mainTask.cancel();

package/src/llm/utils.ts

@@ -323,9 +323,14 @@ export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): Di
   };
 }
 
-export function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {
+export function toJsonSchema(
+  schema: ToolInputSchema<any>,
+  isOpenai: boolean = true,
+  strict: boolean = false,
+): JSONSchema7 {
   if (isZodSchema(schema)) {
-    return zodSchemaToJsonSchema(schema, isOpenai);
+    return zodSchemaToJsonSchema(schema, isOpenai, strict);
   }
+
   return schema as JSONSchema7;
 }

package/src/llm/zod-utils.test.ts

@@ -260,6 +260,107 @@ describe('Zod Utils', () => {
         expect(jsonSchema7).toHaveProperty('properties');
       });
     });
+
+    describe('strict parameter', () => {
+      it('should produce strict JSON schema with strict: true', () => {
+        const schema = z4.object({
+          name: z4.string(),
+          age: z4.number(),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle nullable fields in strict mode', () => {
+        const schema = z4.object({
+          required: z4.string(),
+          optional: z4.string().nullable(),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle default values in strict mode', () => {
+        const schema = z4.object({
+          name: z4.string(),
+          role: z4.string().default('user'),
+          active: z4.boolean().default(true),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle nested objects in strict mode', () => {
+        const schema = z4.object({
+          user: z4.object({
+            name: z4.string(),
+            email: z4.string().nullable(),
+          }),
+          metadata: z4.object({
+            created: z4.string(),
+          }),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle arrays in strict mode', () => {
+        const schema = z4.object({
+          tags: z4.array(z4.string()),
+          numbers: z4.array(z4.number()),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should handle v3 schemas in strict mode', () => {
+        const schema = z3.object({
+          name: z3.string(),
+          age: z3.number().optional(),
+        });
+
+        const strictSchema = zodSchemaToJsonSchema(schema, true, true);
+        expect(strictSchema).toMatchSnapshot();
+      });
+
+      it('should throw error when using .optional() without .nullable() in strict mode', () => {
+        const schema = z4.object({
+          required: z4.string(),
+          optional: z4.string().optional(),
+        });
+
+        expect(() => zodSchemaToJsonSchema(schema, true, true)).toThrow(
+          /uses `.optional\(\)` without `.nullable\(\)` which is not supported by the API/,
+        );
+      });
+
+      it('should throw error for nested .optional() fields in strict mode', () => {
+        const schema = z4.object({
+          user: z4.object({
+            name: z4.string(),
+            email: z4.string().optional(),
+          }),
+        });
+
+        expect(() => zodSchemaToJsonSchema(schema, true, true)).toThrow(
+          /uses `.optional\(\)` without `.nullable\(\)` which is not supported by the API/,
+        );
+      });
+
+      it('should NOT throw error when using .optional() in non-strict mode', () => {
+        const schema = z4.object({
+          required: z4.string(),
+          optional: z4.string().optional(),
+        });
+
+        expect(() => zodSchemaToJsonSchema(schema, true, false)).not.toThrow();
+      });
+    });
   });
 
   describe('parseZodSchema', () => {

package/src/llm/zod-utils.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { JSONSchema7 } from 'json-schema';
+import { toStrictJsonSchema } from 'openai/lib/transform';
 import { zodToJsonSchema as zodToJsonSchemaV3 } from 'zod-to-json-schema';
 import type * as z3 from 'zod/v3';
 import * as z4 from 'zod/v4';
@@ -101,12 +102,18 @@ export function isZodObjectSchema(schema: ZodSchema): boolean {
  * @param isOpenai - Whether to use OpenAI-specific formatting (default: true)
  * @returns A JSON Schema representation of the Zod schema
  */
-export function zodSchemaToJsonSchema(schema: ZodSchema, isOpenai: boolean = true): JSONSchema7 {
+export function zodSchemaToJsonSchema(
+  schema: ZodSchema,
+  isOpenai: boolean = true,
+  strict: boolean = false,
+): JSONSchema7 {
+  let result: JSONSchema7;
+
   if (isZod4Schema(schema)) {
     // Zod v4 has native toJSONSchema support
     // Configuration adapted from Vercel AI SDK to support OpenAPI conversion for Google
     // Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L255-L258
-    return z4.toJSONSchema(schema, {
+    result = z4.toJSONSchema(schema, {
       target: 'draft-7',
       io: 'output',
       reused: 'inline', // Don't use references by default (to support openapi conversion for google)
@@ -115,11 +122,13 @@ export function zodSchemaToJsonSchema(schema: ZodSchema, isOpenai: boolean = tru
     // Zod v3 requires the zod-to-json-schema library
     // Configuration adapted from Vercel AI SDK
     // $refStrategy: 'none' is equivalent to v4's reused: 'inline'
-    return zodToJsonSchemaV3(schema, {
+    result = zodToJsonSchemaV3(schema, {
       target: isOpenai ? 'openAi' : 'jsonSchema7',
       $refStrategy: 'none', // Don't use references by default (to support openapi conversion for google)
     }) as JSONSchema7;
   }
+
+  return strict ? (toStrictJsonSchema(result) as JSONSchema7) : result;
 }
 
 /**

package/src/stt/stt.ts

@@ -204,7 +204,8 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
               options: { retryable: false },
             });
           } else {
-            this.emitError({ error, recoverable: true });
+            // Don't emit error event for recoverable errors during retry loop
+            // to avoid ERR_UNHANDLED_ERROR or premature session termination
             this.logger.warn(
               { tts: this.#stt.label, attempt: i + 1, error },
               `failed to recognize speech, retrying in ${retryInterval}s`,

package/src/tts/tts.ts

@@ -5,7 +5,7 @@ import type { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import { EventEmitter } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
-import { APIConnectionError, APIStatusError } from '../_exceptions.js';
+import { APIConnectionError, APIError } from '../_exceptions.js';
 import { log } from '../log.js';
 import type { TTSMetrics } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
@@ -161,7 +161,7 @@ export abstract class SynthesizeStream
       try {
         return await this.run();
       } catch (error) {
-        if (error instanceof APIStatusError) {
+        if (error instanceof APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
@@ -174,7 +174,8 @@ export abstract class SynthesizeStream
               options: { retryable: false },
             });
           } else {
-            this.emitError({ error, recoverable: true });
+            // Don't emit error event for recoverable errors during retry loop
+            // to avoid ERR_UNHANDLED_ERROR or premature session termination
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
               `failed to synthesize speech, retrying in  ${retryInterval}s`,
@@ -388,7 +389,7 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
       try {
         return await this.run();
       } catch (error) {
-        if (error instanceof APIStatusError) {
+        if (error instanceof APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
@@ -401,7 +402,8 @@ export abstract class ChunkedStream implements AsyncIterableIterator<Synthesized
               options: { retryable: false },
             });
           } else {
-            this.emitError({ error, recoverable: true });
+            // Don't emit error event for recoverable errors during retry loop
+            // to avoid ERR_UNHANDLED_ERROR or premature session termination
             this.logger.warn(
               { tts: this.#tts.label, attempt: i + 1, error },
               `failed to generate TTS completion, retrying in ${retryInterval}s`,

package/src/utils.ts

@@ -15,6 +15,23 @@ import { TransformStream, type TransformStreamDefaultController } from 'node:str
 import { v4 as uuidv4 } from 'uuid';
 import { log } from './log.js';
 
+/**
+ * Recursively expands all nested properties of a type,
+ * resolving aliases so as to inspect the real shape in IDE.
+ */
+// eslint-disable-next-line @typescript-eslint/ban-types
+export type Expand<T> = T extends Function
+  ? T
+  : T extends object
+    ? T extends Array<infer U>
+      ? Array<Expand<U>>
+      : T extends Map<infer K, infer V>
+        ? Map<Expand<K>, Expand<V>>
+        : T extends Set<infer M>
+          ? Set<Expand<M>>
+          : { [K in keyof T]: Expand<T[K]> }
+    : T;
+
 /** Union of a single and a list of {@link AudioFrame}s */
 export type AudioBuffer = AudioFrame[] | AudioFrame;