@livekit/agents 1.1.0 → 1.2.0

package/src/llm/provider_format/utils.ts

@@ -56,12 +56,14 @@ class ChatItemGroup {
   }
 
   removeInvalidToolCalls() {
-    if (this.toolCalls.length === this.toolOutputs.length) {
-      return;
-    }
-
     const toolCallIds = new Set(this.toolCalls.map((call) => call.callId));
     const toolOutputIds = new Set(this.toolOutputs.map((output) => output.callId));
+    const sameIds =
+      toolCallIds.size === toolOutputIds.size &&
+      [...toolCallIds].every((id) => toolOutputIds.has(id));
+    if (this.toolCalls.length === this.toolOutputs.length && sameIds) {
+      return;
+    }
 
     // intersection of tool call ids and tool output ids
     const validCallIds = intersection(toolCallIds, toolOutputIds);
@@ -133,7 +135,11 @@ export function groupToolCalls(chatCtx: ChatContext) {
 
     if (isAssistantMessage || isFunctionCall) {
       // only assistant messages and function calls can be grouped
-      const groupId = item.id.split('/')[0]!;
+      // For function calls, use group_id if available (for parallel function calls),
+      // otherwise fall back to id-based grouping for backwards compatibility
+      const groupId =
+        item.type === 'function_call' && item.groupId ? item.groupId : item.id.split('/')[0]!;
+
       if (itemGroups[groupId] === undefined) {
         itemGroups[groupId] = ChatItemGroup.create();
 

package/src/llm/realtime.ts

@@ -6,6 +6,7 @@ import { EventEmitter } from 'events';
 import type { ReadableStream } from 'node:stream/web';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { Task } from '../utils.js';
+import type { TimedString } from '../voice/io.js';
 import type { ChatContext, FunctionCall } from './chat_context.js';
 import type { ToolChoice, ToolContext } from './tool_context.js';
 
@@ -17,14 +18,20 @@ export interface InputSpeechStoppedEvent {
 
 export interface MessageGeneration {
   messageId: string;
-  textStream: ReadableStream<string>;
+  /**
+   * Text stream that may contain plain strings or TimedString objects with timestamps.
+   */
+  textStream: ReadableStream<string | TimedString>;
   audioStream: ReadableStream<AudioFrame>;
+  modalities?: Promise<('text' | 'audio')[]>;
 }
 
 export interface GenerationCreatedEvent {
   messageStream: ReadableStream<MessageGeneration>;
   functionStream: ReadableStream<FunctionCall>;
   userInitiated: boolean;
+  /** Response ID for correlating metrics with spans */
+  responseId?: string;
 }
 
 export interface RealtimeModelError {
@@ -40,6 +47,8 @@ export interface RealtimeCapabilities {
   turnDetection: boolean;
   userTranscription: boolean;
   autoToolReplyGeneration: boolean;
+  audioOutput: boolean;
+  manualFunctionCalls: boolean;
 }
 
 export interface InputTranscriptionCompleted {
@@ -61,6 +70,13 @@ export abstract class RealtimeModel {
     return this._capabilities;
   }
 
+  /** The model name/identifier used by this realtime model */
+  abstract get model(): string;
+
+  get provider(): string {
+    return 'unknown';
+  }
+
   abstract session(): RealtimeSession;
 
   abstract close(): Promise<void>;
@@ -121,7 +137,12 @@ export abstract class RealtimeSession extends EventEmitter {
   /**
    * Truncate the message at the given audio end time
    */
-  abstract truncate(options: { messageId: string; audioEndMs: number }): Promise<void>;
+  abstract truncate(options: {
+    messageId: string;
+    audioEndMs: number;
+    modalities?: ('text' | 'audio')[];
+    audioTranscript?: string;
+  }): Promise<void>;
 
   async close(): Promise<void> {
     this._mainTask.cancel();

package/src/llm/remote_chat_context.ts

@@ -6,9 +6,9 @@ import type { ChatItem } from './chat_context.js';
 
 export interface RemoteChatItem {
   item: ChatItem;
-  /* @internal */
+  /** @internal */
   _prev?: RemoteChatItem | null;
-  /* @internal */
+  /** @internal */
   _next?: RemoteChatItem | null;
 }
 

package/src/llm/tool_context.test.ts

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 import { describe, expect, it } from 'vitest';
 import { z } from 'zod';
+import * as z3 from 'zod/v3';
+import * as z4 from 'zod/v4';
 import { type ToolOptions, tool } from './tool_context.js';
 import { createToolOptions, oaiParams } from './utils.js';
 
@@ -164,12 +166,43 @@ describe('Tool Context', () => {
         expect(simpleAction.type).toBe('function');
         expect(simpleAction.description).toBe('Perform a simple action');
         expect(simpleAction.parameters).toBeDefined();
-        expect(simpleAction.parameters._def.typeName).toBe('ZodObject');
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        expect((simpleAction.parameters as any)._def.typeName).toBe('ZodObject');
 
         const result = await simpleAction.execute({}, createToolOptions('123'));
         expect(result).toBe('Action performed');
       });
 
+      it('should support .optional() fields in tool parameters', async () => {
+        const weatherTool = tool({
+          description: 'Get weather information',
+          parameters: z.object({
+            location: z.string().describe('The city or location').optional(),
+            units: z.enum(['celsius', 'fahrenheit']).describe('Temperature units').optional(),
+          }),
+          execute: async ({ location, units }) => {
+            const loc = location ?? 'Unknown';
+            const unit = units ?? 'celsius';
+            return `Weather in ${loc} (${unit})`;
+          },
+        });
+
+        expect(weatherTool.type).toBe('function');
+        expect(weatherTool.description).toBe('Get weather information');
+
+        const result1 = await weatherTool.execute(
+          { location: 'London', units: 'celsius' },
+          createToolOptions('123'),
+        );
+        expect(result1).toBe('Weather in London (celsius)');
+
+        const result2 = await weatherTool.execute({}, createToolOptions('123'));
+        expect(result2).toBe('Weather in Unknown (celsius)');
+
+        const result3 = await weatherTool.execute({ location: 'Paris' }, createToolOptions('123'));
+        expect(result3).toBe('Weather in Paris (celsius)');
+      });
+
       it('should handle tools with context but no parameters', async () => {
         const greetUser = tool({
           description: 'Greet the current user',
@@ -194,5 +227,181 @@ describe('Tool Context', () => {
         expect(result).toBe('Tool call ID: test-id-456');
       });
     });
+
+    describe('Zod v3 and v4 compatibility', () => {
+      it('should work with Zod v3 schemas', async () => {
+        const v3Tool = tool({
+          description: 'A tool using Zod v3 schema',
+          parameters: z3.object({
+            name: z3.string(),
+            count: z3.number(),
+          }),
+          execute: async ({ name, count }) => {
+            return `${name}: ${count}`;
+          },
+        });
+
+        const result = await v3Tool.execute(
+          { name: 'Test', count: 42 },
+          createToolOptions('v3-test'),
+        );
+        expect(result).toBe('Test: 42');
+      });
+
+      it('should work with Zod v4 schemas', async () => {
+        const v4Tool = tool({
+          description: 'A tool using Zod v4 schema',
+          parameters: z4.object({
+            name: z4.string(),
+            count: z4.number(),
+          }),
+          execute: async ({ name, count }) => {
+            return `${name}: ${count}`;
+          },
+        });
+
+        const result = await v4Tool.execute(
+          { name: 'Test', count: 42 },
+          createToolOptions('v4-test'),
+        );
+        expect(result).toBe('Test: 42');
+      });
+
+      it('should handle v4 schemas with optional fields', async () => {
+        const v4Tool = tool({
+          description: 'Tool with optional field using v4',
+          parameters: z4.object({
+            required: z4.string(),
+            optional: z4.string().optional(),
+          }),
+          execute: async ({ required, optional }) => {
+            return optional ? `${required} - ${optional}` : required;
+          },
+        });
+
+        const result1 = await v4Tool.execute({ required: 'Hello' }, createToolOptions('test-1'));
+        expect(result1).toBe('Hello');
+
+        const result2 = await v4Tool.execute(
+          { required: 'Hello', optional: 'World' },
+          createToolOptions('test-2'),
+        );
+        expect(result2).toBe('Hello - World');
+      });
+
+      it('should handle v4 enum schemas', async () => {
+        const v4Tool = tool({
+          description: 'Tool with enum using v4',
+          parameters: z4.object({
+            color: z4.enum(['red', 'blue', 'green']),
+          }),
+          execute: async ({ color }) => {
+            return `Selected color: ${color}`;
+          },
+        });
+
+        const result = await v4Tool.execute({ color: 'blue' }, createToolOptions('test-enum'));
+        expect(result).toBe('Selected color: blue');
+      });
+
+      it('should handle v4 array schemas', async () => {
+        const v4Tool = tool({
+          description: 'Tool with array using v4',
+          parameters: z4.object({
+            tags: z4.array(z4.string()),
+          }),
+          execute: async ({ tags }) => {
+            return `Tags: ${tags.join(', ')}`;
+          },
+        });
+
+        const result = await v4Tool.execute(
+          { tags: ['nodejs', 'typescript', 'testing'] },
+          createToolOptions('test-array'),
+        );
+        expect(result).toBe('Tags: nodejs, typescript, testing');
+      });
+
+      it('should handle v4 nested object schemas', async () => {
+        const v4Tool = tool({
+          description: 'Tool with nested object using v4',
+          parameters: z4.object({
+            user: z4.object({
+              name: z4.string(),
+              email: z4.string(),
+            }),
+          }),
+          execute: async ({ user }) => {
+            return `${user.name} (${user.email})`;
+          },
+        });
+
+        const result = await v4Tool.execute(
+          { user: { name: 'John Doe', email: 'john@example.com' } },
+          createToolOptions('test-nested'),
+        );
+        expect(result).toBe('John Doe (john@example.com)');
+      });
+    });
+
+    describe('oaiParams with v4 schemas', () => {
+      it('should convert v4 basic object schema', () => {
+        const schema = z4.object({
+          name: z4.string().describe('User name'),
+          age: z4.number().describe('User age'),
+        });
+
+        const result = oaiParams(schema);
+
+        expect(result.type).toBe('object');
+        expect(result.properties).toHaveProperty('name');
+        expect(result.properties).toHaveProperty('age');
+        expect(result.required).toContain('name');
+        expect(result.required).toContain('age');
+      });
+
+      it('should handle v4 optional fields', () => {
+        const schema = z4.object({
+          required: z4.string(),
+          optional: z4.string().optional(),
+        });
+
+        const result = oaiParams(schema);
+
+        expect(result.required).toContain('required');
+        expect(result.required).not.toContain('optional');
+      });
+
+      it('should handle v4 enum fields', () => {
+        const schema = z4.object({
+          status: z4.enum(['pending', 'approved', 'rejected']),
+        });
+
+        const result = oaiParams(schema);
+
+        const properties = result.properties as Record<string, Record<string, unknown>>;
+        expect(properties.status?.enum).toEqual(['pending', 'approved', 'rejected']);
+      });
+
+      it('should handle v4 array fields', () => {
+        const schema = z4.object({
+          items: z4.array(z4.string()),
+        });
+
+        const result = oaiParams(schema);
+
+        const properties = result.properties as Record<string, any>;
+        expect(
+          properties.items && typeof properties.items === 'object'
+            ? properties.items.type
+            : undefined,
+        ).toBe('array');
+        expect(
+          properties.items && properties.items.items && typeof properties.items.items === 'object'
+            ? properties.items.items.type
+            : undefined,
+        ).toBe('string');
+      });
+    });
   });
 });

package/src/llm/tool_context.ts

@@ -2,9 +2,10 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { JSONSchema7 } from 'json-schema';
-import { ZodObject, ZodType, z } from 'zod';
+import { z } from 'zod';
 import type { Agent } from '../voice/agent.js';
 import type { RunContext, UnknownUserData } from '../voice/run_context.js';
+import { isZodObjectSchema, isZodSchema } from './zod-utils.js';
 
 // heavily inspired by Vercel AI's `tool()`:
 // https://github.com/vercel/ai/blob/3b0983b/packages/ai/core/tool/tool.ts
@@ -23,9 +24,38 @@ export type JSONObject = {
   [key: string]: JSONValue;
 };
 
-// TODO(AJS-111): support Zod cross-version compatibility, raw JSON schema, both strict and non-strict versions
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type ToolInputSchema<T extends JSONObject> = ZodObject<any, any, any, T, T> | JSONSchema7;
+// Supports both Zod v3 and v4 schemas, as well as raw JSON schema
+// Adapted from Vercel AI SDK's FlexibleSchema approach
+// Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L67-L70
+//
+// Vercel uses StandardSchemaV1 from @standard-schema/spec package.
+// We use a simpler approach by directly checking for schema properties:
+// - Zod v3: Has `_output` property
+// - Zod v4: Implements Standard Schema spec with `~standard` property
+// - JSON Schema: Plain object fallback
+export type ToolInputSchema<T = JSONObject> =
+  | {
+      // Zod v3 schema - has _output property for type inference
+      _output: T;
+    }
+  | {
+      // Zod v4 schema (Standard Schema) - has ~standard property
+      '~standard': {
+        types?: { output: T };
+      };
+    }
+  | JSONSchema7;
+
+/**
+ * Infer the output type from a ToolInputSchema.
+ * Adapted from Vercel AI SDK's InferSchema type.
+ * Source: https://github.com/vercel/ai/blob/main/packages/provider-utils/src/schema.ts#L72-L79
+ */
+export type InferToolInput<T> = T extends { _output: infer O }
+  ? O
+  : T extends { '~standard': { types?: { output: infer O } } }
+    ? O
+    : any; // eslint-disable-line @typescript-eslint/no-explicit-any -- Fallback type for JSON Schema objects without type inference
 
 export type ToolType = 'function' | 'provider-defined';
 
@@ -50,6 +80,13 @@ export class ToolError extends Error {
   }
 }
 
+export const ToolFlag = {
+  NONE: 0,
+  IGNORE_ON_ENTER: 1 << 0,
+} as const;
+
+export type ToolFlag = (typeof ToolFlag)[keyof typeof ToolFlag];
+
 export interface AgentHandoff {
   /**
    * The agent to handoff to.
@@ -148,34 +185,93 @@ export interface FunctionTool<
    */
   execute: ToolExecuteFunction<Parameters, UserData, Result>;
 
+  flags: number;
+
   [FUNCTION_TOOL_SYMBOL]: true;
 }
 
 // TODO(AJS-112): support provider-defined tools in the future)
 export type ToolContext<UserData = UnknownUserData> = {
-  [name: string]: FunctionTool<any, UserData, any>; // eslint-disable-line @typescript-eslint/no-explicit-any
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Generic tool registry needs to accept any parameter/result types
+  [name: string]: FunctionTool<any, UserData, any>;
 };
 
+export function isSameToolContext(ctx1: ToolContext, ctx2: ToolContext): boolean {
+  const toolNames = new Set(Object.keys(ctx1));
+  const toolNames2 = new Set(Object.keys(ctx2));
+
+  if (toolNames.size !== toolNames2.size) {
+    return false;
+  }
+
+  for (const name of toolNames) {
+    if (!toolNames2.has(name)) {
+      return false;
+    }
+
+    const tool1 = ctx1[name];
+    const tool2 = ctx2[name];
+
+    if (!tool1 || !tool2) {
+      return false;
+    }
+
+    if (tool1.description !== tool2.description) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+export function isSameToolChoice(choice1: ToolChoice | null, choice2: ToolChoice | null): boolean {
+  if (choice1 === choice2) {
+    return true;
+  }
+  if (choice1 === null || choice2 === null) {
+    return false;
+  }
+  if (typeof choice1 === 'string' && typeof choice2 === 'string') {
+    return choice1 === choice2;
+  }
+  if (typeof choice1 === 'object' && typeof choice2 === 'object') {
+    return choice1.type === choice2.type && choice1.function.name === choice2.function.name;
+  }
+  return false;
+}
+
 /**
- * Create a function tool.
- *
- * @param description - The description of the tool.
- * @param parameters - The schema of the input that the tool expects. If not provided, defaults to z.object({}).
- * @param execute - The function that is called with the arguments from the tool call and produces a result.
+ * Create a function tool with inferred parameters from the schema.
  */
 export function tool<
-  Parameters extends JSONObject = Record<string, never>,
+  Schema extends ToolInputSchema<any>, // eslint-disable-line @typescript-eslint/no-explicit-any -- Generic constraint needs to accept any JSONObject type
   UserData = UnknownUserData,
   Result = unknown,
 >({
   description,
   parameters,
   execute,
+  flags,
 }: {
   description: string;
-  parameters?: ToolInputSchema<Parameters>;
-  execute: ToolExecuteFunction<Parameters, UserData, Result>;
-}): FunctionTool<Parameters, UserData, Result>;
+  parameters: Schema;
+  execute: ToolExecuteFunction<InferToolInput<Schema>, UserData, Result>;
+  flags?: number;
+}): FunctionTool<InferToolInput<Schema>, UserData, Result>;
+
+/**
+ * Create a function tool without parameters.
+ */
+export function tool<UserData = UnknownUserData, Result = unknown>({
+  description,
+  execute,
+  flags,
+}: {
+  description: string;
+  parameters?: never;
+  execute: ToolExecuteFunction<Record<string, never>, UserData, Result>;
+  flags?: number;
+}): FunctionTool<Record<string, never>, UserData, Result>;
 
 /**
  * Create a provider-defined tool.
@@ -197,12 +293,13 @@ export function tool(tool: any): any {
     // Default parameters to z.object({}) if not provided
     const parameters = tool.parameters ?? z.object({});
 
-    // if parameters is not zod object, throw an error
-    if (parameters instanceof ZodType && parameters._def.typeName !== 'ZodObject') {
+    // if parameters is a Zod schema, ensure it's an object schema
+    if (isZodSchema(parameters) && !isZodObjectSchema(parameters)) {
       throw new Error('Tool parameters must be a Zod object schema (z.object(...))');
     }
 
-    if (!(parameters instanceof ZodObject) && !(typeof parameters === 'object')) {
+    // Ensure parameters is either a Zod schema or a plain object (JSON schema)
+    if (!isZodSchema(parameters) && !(typeof parameters === 'object')) {
       throw new Error('Tool parameters must be a Zod object schema or a raw JSON schema');
     }
 
@@ -211,6 +308,7 @@ export function tool(tool: any): any {
       description: tool.description,
       parameters,
       execute: tool.execute,
+      flags: tool.flags ?? ToolFlag.NONE,
       [TOOL_SYMBOL]: true,
       [FUNCTION_TOOL_SYMBOL]: true,
     };

package/src/llm/utils.ts

@@ -4,8 +4,6 @@
 import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
 import type { JSONSchema7 } from 'json-schema';
 import sharp from 'sharp';
-import { ZodObject } from 'zod';
-import { zodToJsonSchema } from 'zod-to-json-schema';
 import type { UnknownUserData } from '../voice/run_context.js';
 import type { ChatContext } from './chat_context.js';
 import {
@@ -15,6 +13,7 @@ import {
   type ImageContent,
 } from './chat_context.js';
 import type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';
+import { isZodSchema, parseZodSchema, zodSchemaToJsonSchema } from './zod-utils.js';
 
 export interface SerializedImage {
   inferenceDetail: 'auto' | 'high' | 'low';
@@ -151,15 +150,10 @@ export const createToolOptions = <UserData extends UnknownUserData>(
 
 /** @internal */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const oaiParams = (
-  p: ZodObject<any>,
-  isOpenai: boolean = true,
-): OpenAIFunctionParameters => {
+export const oaiParams = (schema: any, isOpenai: boolean = true): OpenAIFunctionParameters => {
   // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts
-  const { properties, required, additionalProperties } = zodToJsonSchema(p, {
-    // note: openai mode breaks various gemini conversions
-    target: isOpenai ? 'openAi' : 'jsonSchema7',
-  }) as OpenAIFunctionParameters;
+  const jsonSchema = zodSchemaToJsonSchema(schema, isOpenai);
+  const { properties, required, additionalProperties } = jsonSchema as OpenAIFunctionParameters;
 
   return {
     type: 'object',
@@ -209,8 +203,17 @@ export async function executeToolCall(
 
   // Ensure valid arguments schema
   try {
-    if (tool.parameters instanceof ZodObject) {
-      params = tool.parameters.parse(args);
+    if (isZodSchema(tool.parameters)) {
+      const result = await parseZodSchema<object>(tool.parameters, args);
+      if (result.success) {
+        params = result.data;
+      } else {
+        return FunctionCallOutput.create({
+          callId: toolCall.callId,
+          output: `Arguments parsing failed: ${result.error}`,
+          isError: true,
+        });
+      }
     } else {
       params = args;
     }
@@ -320,9 +323,14 @@ export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): Di
   };
 }
 
-export function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {
-  if (schema instanceof ZodObject) {
-    return oaiParams(schema, isOpenai);
+export function toJsonSchema(
+  schema: ToolInputSchema<any>,
+  isOpenai: boolean = true,
+  strict: boolean = false,
+): JSONSchema7 {
+  if (isZodSchema(schema)) {
+    return zodSchemaToJsonSchema(schema, isOpenai, strict);
   }
-  return schema;
+
+  return schema as JSONSchema7;
 }