@livekit/agents 0.7.9 → 1.0.0-next.1

package/src/llm/remote_chat_context.ts

@@ -0,0 +1,114 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { ChatContext } from './chat_context.js';
+import type { ChatItem } from './chat_context.js';
+
+export interface RemoteChatItem {
+  item: ChatItem;
+  /* @internal */
+  _prev?: RemoteChatItem | null;
+  /* @internal */
+  _next?: RemoteChatItem | null;
+}
+
+export class RemoteChatContext {
+  private head?: RemoteChatItem | null;
+  private tail?: RemoteChatItem | null;
+  private idToItem: Record<string, RemoteChatItem> = {};
+
+  toChatCtx(): ChatContext {
+    const items: ChatItem[] = [];
+    let currentNode = this.head;
+    while (currentNode) {
+      items.push(currentNode.item);
+      currentNode = currentNode._next;
+    }
+
+    return new ChatContext(items);
+  }
+
+  get(itemId: string): RemoteChatItem | null {
+    return this.idToItem[itemId] ?? null;
+  }
+
+  /**
+   * Insert `message` after the node with ID `previousItemId`.
+   * If `previousItemId` is undefined, insert at the head.
+   * @param previousItemId - The ID of the item after which to insert the new item.
+   * @param message - The item to insert.
+   */
+  insert(previousItemId: string | undefined, message: ChatItem): void {
+    const itemId = message.id;
+
+    if (itemId in this.idToItem) {
+      throw new Error(`Item with ID ${itemId} already exists.`);
+    }
+
+    const newNode: RemoteChatItem = { item: message };
+
+    if (!previousItemId) {
+      if (this.head) {
+        newNode._next = this.head;
+        this.head._prev = newNode;
+      } else {
+        this.tail = newNode;
+      }
+      this.head = newNode;
+      this.idToItem[itemId] = newNode;
+      return;
+    }
+
+    const prevNode = this.idToItem[previousItemId];
+    if (!prevNode) {
+      throw new Error(`previousItemId ${previousItemId} not found`);
+    }
+
+    newNode._prev = prevNode;
+    newNode._next = prevNode._next;
+
+    prevNode._next = newNode;
+
+    if (newNode._next) {
+      newNode._next._prev = newNode;
+    } else {
+      this.tail = newNode;
+    }
+
+    this.idToItem[itemId] = newNode;
+  }
+
+  delete(itemId: string): void {
+    const node = this.idToItem[itemId];
+    if (!node) {
+      throw new Error(`Item with ID ${itemId} not found`);
+    }
+
+    const prevNode = node._prev;
+    const nextNode = node._next;
+
+    if (this.head === node) {
+      this.head = nextNode;
+      if (this.head) {
+        this.head._prev = undefined;
+      }
+    } else {
+      if (prevNode) {
+        prevNode._next = nextNode;
+      }
+    }
+
+    if (this.tail === node) {
+      this.tail = prevNode;
+      if (this.tail) {
+        this.tail._next = undefined;
+      }
+    } else {
+      if (nextNode) {
+        nextNode._prev = prevNode;
+      }
+    }
+
+    delete this.idToItem[itemId];
+  }
+}

package/src/llm/tool_context.test.ts

@@ -0,0 +1,198 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { z } from 'zod';
+import { type ToolOptions, tool } from './tool_context.js';
+import { createToolOptions, oaiParams } from './utils.js';
+
+describe('Tool Context', () => {
+  describe('oaiParams', () => {
+    it('should handle basic object schema', () => {
+      const schema = z.object({
+        name: z.string().describe('The user name'),
+        age: z.number().describe('The user age'),
+      });
+
+      const result = oaiParams(schema);
+
+      expect(result).toMatchSnapshot();
+    });
+
+    it('should handle enum fields', () => {
+      const schema = z.object({
+        color: z.enum(['red', 'blue', 'green']).describe('Choose a color'),
+      });
+
+      const result = oaiParams(schema);
+
+      expect(result).toMatchSnapshot();
+    });
+
+    it('should handle array fields', () => {
+      const schema = z.object({
+        tags: z.array(z.string()).describe('List of tags'),
+      });
+
+      const result = oaiParams(schema);
+
+      expect(result).toMatchSnapshot();
+    });
+
+    it('should handle array of enums', () => {
+      const schema = z.object({
+        colors: z.array(z.enum(['red', 'blue', 'green'])).describe('List of colors'),
+      });
+
+      const result = oaiParams(schema);
+
+      expect(result).toMatchSnapshot();
+    });
+
+    it('should handle optional fields', () => {
+      const schema = z.object({
+        name: z.string().describe('The user name'),
+        age: z.number().optional().describe('The user age'),
+      });
+
+      const result = oaiParams(schema);
+
+      expect(result).toMatchSnapshot();
+    });
+
+    it('should handle fields without descriptions', () => {
+      const schema = z.object({
+        name: z.string(),
+        age: z.number(),
+      });
+
+      const result = oaiParams(schema);
+
+      expect(result).toMatchSnapshot();
+    });
+  });
+
+  describe('tool', () => {
+    it('should create and execute a basic core tool', async () => {
+      const getWeather = tool({
+        description: 'Get the weather for a given location',
+        parameters: z.object({
+          location: z.string(),
+        }),
+        execute: async ({ location }, { ctx }: ToolOptions<{ name: string }>) => {
+          return `The weather in ${location} is sunny, ${ctx.userData.name}`;
+        },
+      });
+
+      const result = await getWeather.execute(
+        { location: 'San Francisco' },
+        createToolOptions('123', { name: 'John' }),
+      );
+      expect(result).toBe('The weather in San Francisco is sunny, John');
+    });
+
+    it('should properly type a callable function', async () => {
+      const testFunction = tool({
+        description: 'Test function',
+        parameters: z.object({
+          name: z.string().describe('The user name'),
+          age: z.number().describe('The user age'),
+        }),
+        execute: async (args) => {
+          return `${args.name} is ${args.age} years old`;
+        },
+      });
+
+      const result = await testFunction.execute(
+        { name: 'John', age: 30 },
+        createToolOptions('123'),
+      );
+      expect(result).toBe('John is 30 years old');
+    });
+
+    it('should handle async execution', async () => {
+      const testFunction = tool({
+        description: 'Async test function',
+        parameters: z.object({
+          delay: z.number().describe('Delay in milliseconds'),
+        }),
+        execute: async (args) => {
+          await new Promise((resolve) => setTimeout(resolve, args.delay));
+          return args.delay;
+        },
+      });
+
+      const start = Date.now();
+      const result = await testFunction.execute({ delay: 100 }, createToolOptions('123'));
+      const duration = Date.now() - start;
+
+      expect(result).toBe(100);
+      expect(duration).toBeGreaterThanOrEqual(95); // Allow for small timing variations
+    });
+
+    describe('nested array support', () => {
+      it('should handle nested array fields', () => {
+        const schema = z.object({
+          items: z.array(
+            z.object({
+              name: z.string().describe('the item name'),
+              modifiers: z
+                .array(
+                  z.object({
+                    modifier_name: z.string(),
+                    modifier_value: z.string(),
+                  }),
+                )
+                .describe('list of the modifiers applied on this item, such as size'),
+            }),
+          ),
+        });
+        const result = oaiParams(schema);
+        expect(result).toMatchSnapshot();
+      });
+    });
+
+    describe('optional parameters', () => {
+      it('should create a tool without parameters', async () => {
+        const simpleAction = tool({
+          description: 'Perform a simple action',
+          execute: async () => {
+            return 'Action performed';
+          },
+        });
+
+        expect(simpleAction.type).toBe('function');
+        expect(simpleAction.description).toBe('Perform a simple action');
+        expect(simpleAction.parameters).toBeDefined();
+        expect(simpleAction.parameters._def.typeName).toBe('ZodObject');
+
+        const result = await simpleAction.execute({}, createToolOptions('123'));
+        expect(result).toBe('Action performed');
+      });
+
+      it('should handle tools with context but no parameters', async () => {
+        const greetUser = tool({
+          description: 'Greet the current user',
+          execute: async (_, { ctx }: ToolOptions<{ username: string }>) => {
+            return `Hello, ${ctx.userData.username}!`;
+          },
+        });
+
+        const result = await greetUser.execute({}, createToolOptions('123', { username: 'Alice' }));
+        expect(result).toBe('Hello, Alice!');
+      });
+
+      it('should create a tool that accesses tool call id without parameters', async () => {
+        const getCallId = tool({
+          description: 'Get the current tool call ID',
+          execute: async (_, { toolCallId }) => {
+            return `Tool call ID: ${toolCallId}`;
+          },
+        });
+
+        const result = await getCallId.execute({}, createToolOptions('test-id-456'));
+        expect(result).toBe('Tool call ID: test-id-456');
+      });
+    });
+  });
+});

package/src/llm/tool_context.ts

@@ -0,0 +1,259 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { JSONSchema7 } from 'json-schema';
+import { ZodObject, ZodType, z } from 'zod';
+import type { Agent } from '../voice/agent.js';
+import type { RunContext, UnknownUserData } from '../voice/run_context.js';
+
+// heavily inspired by Vercel AI's `tool()`:
+// https://github.com/vercel/ai/blob/3b0983b/packages/ai/core/tool/tool.ts
+
+const TOOL_SYMBOL = Symbol('tool');
+const FUNCTION_TOOL_SYMBOL = Symbol('function_tool');
+const PROVIDER_DEFINED_TOOL_SYMBOL = Symbol('provider_defined_tool');
+const TOOL_ERROR_SYMBOL = Symbol('tool_error');
+const HANDOFF_SYMBOL = Symbol('handoff');
+
+export type JSONValue = null | string | number | boolean | JSONObject | JSONArray;
+
+export type JSONArray = JSONValue[];
+
+export type JSONObject = {
+  [key: string]: JSONValue;
+};
+
+// TODO(AJS-111): support Zod cross-version compatibility, raw JSON schema, both strict and non-strict versions
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type ToolInputSchema<T extends JSONObject> = ZodObject<any, any, any, T, T> | JSONSchema7;
+
+export type ToolType = 'function' | 'provider-defined';
+
+export type ToolChoice =
+  | 'auto'
+  | 'none'
+  | 'required'
+  | {
+      type: 'function';
+      function: {
+        name: string;
+      };
+    };
+
+export class ToolError extends Error {
+  constructor(message: string) {
+    super(message);
+
+    Object.defineProperty(this, TOOL_ERROR_SYMBOL, {
+      value: true,
+    });
+  }
+}
+
+export interface AgentHandoff {
+  /**
+   * The agent to handoff to.
+   */
+  agent: Agent;
+
+  /**
+   * The return value of the tool.
+   */
+  returns?: any; // eslint-disable-line @typescript-eslint/no-explicit-any
+
+  [HANDOFF_SYMBOL]: true;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function handoff(options: { agent: Agent; returns?: any }): AgentHandoff {
+  return {
+    agent: options.agent,
+    returns: options.returns,
+    [HANDOFF_SYMBOL]: true,
+  };
+}
+
+export interface ToolOptions<UserData = UnknownUserData> {
+  /**
+   * RunContext for the current agent session.
+   */
+  ctx: RunContext<UserData>;
+
+  /**
+   * The ID of the tool call.
+   */
+  toolCallId: string;
+
+  /**
+   * An optional abort signal that indicates that the overall operation should be aborted.
+   */
+  abortSignal?: AbortSignal;
+}
+
+export type ToolExecuteFunction<
+  Parameters extends JSONObject,
+  UserData = UnknownUserData,
+  Result = unknown,
+> = (args: Parameters, opts: ToolOptions<UserData>) => Promise<Result>;
+
+export interface Tool {
+  /**
+   * The type of the tool.
+   * @internal Either user-defined core tool or provider-defined tool.
+   */
+  type: ToolType;
+
+  [TOOL_SYMBOL]: true;
+}
+
+// TODO(AJS-112): support provider-defined tools
+export interface ProviderDefinedTool extends Tool {
+  type: 'provider-defined';
+
+  /**
+   * The ID of the tool.
+   */
+  id: string;
+
+  /**
+   * The configuration of the tool.
+   */
+  config: Record<string, unknown>;
+
+  [PROVIDER_DEFINED_TOOL_SYMBOL]: true;
+}
+
+export interface FunctionTool<
+  Parameters extends JSONObject,
+  UserData = UnknownUserData,
+  Result = unknown,
+> extends Tool {
+  type: 'function';
+
+  /**
+   * The description of the tool. Will be used by the language model to decide whether to use the tool.
+   */
+  description: string;
+
+  /**
+   * The schema of the input that the tool expects. The language model will use this to generate the input.
+   * It is also used to validate the output of the language model.
+   * Use descriptions to make the input understandable for the language model.
+   */
+  parameters: ToolInputSchema<Parameters>;
+
+  /**
+   * An async function that is called with the arguments from the tool call and produces a result.
+   * It also carries context about current session, user-defined data, and the tool call id, etc.
+   */
+  execute: ToolExecuteFunction<Parameters, UserData, Result>;
+
+  [FUNCTION_TOOL_SYMBOL]: true;
+}
+
+// TODO(AJS-112): support provider-defined tools in the future)
+export type ToolContext<UserData = UnknownUserData> = {
+  [name: string]: FunctionTool<any, UserData, any>; // eslint-disable-line @typescript-eslint/no-explicit-any
+};
+
+/**
+ * Create a function tool.
+ *
+ * @param description - The description of the tool.
+ * @param parameters - The schema of the input that the tool expects. If not provided, defaults to z.object({}).
+ * @param execute - The function that is called with the arguments from the tool call and produces a result.
+ */
+export function tool<
+  Parameters extends JSONObject = Record<string, never>,
+  UserData = UnknownUserData,
+  Result = unknown,
+>({
+  description,
+  parameters,
+  execute,
+}: {
+  description: string;
+  parameters?: ToolInputSchema<Parameters>;
+  execute: ToolExecuteFunction<Parameters, UserData, Result>;
+}): FunctionTool<Parameters, UserData, Result>;
+
+/**
+ * Create a provider-defined tool.
+ *
+ * @param id - The ID of the tool.
+ * @param config - The configuration of the tool.
+ */
+export function tool({
+  id,
+  config,
+}: {
+  id: string;
+  config: Record<string, unknown>;
+}): ProviderDefinedTool;
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function tool(tool: any): any {
+  if (tool.execute !== undefined) {
+    // Default parameters to z.object({}) if not provided
+    const parameters = tool.parameters ?? z.object({});
+
+    // if parameters is not zod object, throw an error
+    if (parameters instanceof ZodType && parameters._def.typeName !== 'ZodObject') {
+      throw new Error('Tool parameters must be a Zod object schema (z.object(...))');
+    }
+
+    if (!(parameters instanceof ZodObject) && !(typeof parameters === 'object')) {
+      throw new Error('Tool parameters must be a Zod object schema or a raw JSON schema');
+    }
+
+    return {
+      type: 'function',
+      description: tool.description,
+      parameters,
+      execute: tool.execute,
+      [TOOL_SYMBOL]: true,
+      [FUNCTION_TOOL_SYMBOL]: true,
+    };
+  }
+
+  if (tool.config !== undefined && tool.id !== undefined) {
+    return {
+      type: 'provider-defined',
+      id: tool.id,
+      config: tool.config,
+      [TOOL_SYMBOL]: true,
+      [PROVIDER_DEFINED_TOOL_SYMBOL]: true,
+    };
+  }
+
+  throw new Error('Invalid tool');
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function isTool(tool: any): tool is Tool {
+  return tool && tool[TOOL_SYMBOL] === true;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function isFunctionTool(tool: any): tool is FunctionTool<any, any, any> {
+  const isTool = tool && tool[TOOL_SYMBOL] === true;
+  const isFunctionTool = tool[FUNCTION_TOOL_SYMBOL] === true;
+  return isTool && isFunctionTool;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function isProviderDefinedTool(tool: any): tool is ProviderDefinedTool {
+  const isTool = tool && tool[TOOL_SYMBOL] === true;
+  const isProviderDefinedTool = tool[PROVIDER_DEFINED_TOOL_SYMBOL] === true;
+  return isTool && isProviderDefinedTool;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function isToolError(error: any): error is ToolError {
+  return error && error[TOOL_ERROR_SYMBOL] === true;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function isAgentHandoff(handoff: any): handoff is AgentHandoff {
+  return handoff && handoff[HANDOFF_SYMBOL] === true;
+}

package/src/llm/tool_context.type.test.ts

@@ -0,0 +1,115 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, expectTypeOf, it } from 'vitest';
+import { z } from 'zod';
+import { type FunctionTool, type ProviderDefinedTool, type ToolOptions, tool } from './index.js';
+
+describe('tool type inference', () => {
+  it('should infer argument type from zod schema', () => {
+    const toolType = tool({
+      description: 'test',
+      parameters: z.object({ number: z.number() }),
+      execute: async () => 'test' as const,
+    });
+
+    expectTypeOf(toolType).toEqualTypeOf<FunctionTool<{ number: number }, unknown, 'test'>>();
+  });
+
+  it('should infer provider defined tool type', () => {
+    const toolType = tool({
+      id: 'code-interpreter',
+      config: {
+        language: 'python',
+      },
+    });
+
+    expectTypeOf(toolType).toEqualTypeOf<ProviderDefinedTool>();
+  });
+
+  it('should infer run context type', () => {
+    const toolType = tool({
+      description: 'test',
+      parameters: z.object({ number: z.number() }),
+      execute: async ({ number }, { ctx }: ToolOptions<{ name: string }>) => {
+        return `The number is ${number}, ${ctx.userData.name}`;
+      },
+    });
+
+    expectTypeOf(toolType).toEqualTypeOf<
+      FunctionTool<{ number: number }, { name: string }, string>
+    >();
+  });
+
+  it('should not accept primitive zod schemas', () => {
+    expect(() => {
+      // @ts-expect-error - Testing that non-object schemas are rejected
+      tool({
+        name: 'test',
+        description: 'test',
+        parameters: z.string(),
+        execute: async () => 'test' as const,
+      });
+    }).toThrowError('Tool parameters must be a Zod object schema (z.object(...))');
+  });
+
+  it('should not accept array schemas', () => {
+    expect(() => {
+      // @ts-expect-error - Testing that array schemas are rejected
+      tool({
+        name: 'test',
+        description: 'test',
+        parameters: z.array(z.string()),
+        execute: async () => 'test' as const,
+      });
+    }).toThrowError('Tool parameters must be a Zod object schema (z.object(...))');
+  });
+
+  it('should not accept union schemas', () => {
+    expect(() => {
+      // @ts-expect-error - Testing that union schemas are rejected
+      tool({
+        name: 'test',
+        description: 'test',
+        parameters: z.union([z.object({ a: z.string() }), z.object({ b: z.number() })]),
+        execute: async () => 'test' as const,
+      });
+    }).toThrowError('Tool parameters must be a Zod object schema (z.object(...))');
+  });
+
+  it('should not accept non-Zod values as parameters', () => {
+    expect(() => {
+      // @ts-expect-error - Testing that non-Zod values are rejected
+      tool({
+        name: 'test',
+        description: 'test',
+        parameters: 'invalid schema',
+        execute: async () => 'test' as const,
+      });
+    }).toThrowError('Tool parameters must be a Zod object schema or a raw JSON schema');
+  });
+
+  it('should infer empty object type when parameters are omitted', () => {
+    const toolType = tool({
+      description: 'Simple action without parameters',
+      execute: async () => 'done' as const,
+    });
+
+    expectTypeOf(toolType).toEqualTypeOf<FunctionTool<Record<string, never>, unknown, 'done'>>();
+  });
+
+  it('should infer correct types with context but no parameters', () => {
+    const toolType = tool({
+      description: 'Action with context',
+      execute: async (args, { ctx }: ToolOptions<{ userId: number }>) => {
+        expectTypeOf(args).toEqualTypeOf<Record<string, never>>();
+        expectTypeOf(ctx.userData.userId).toEqualTypeOf<number>();
+        return ctx.userData.userId;
+      },
+    });
+
+    expectTypeOf(toolType).toEqualTypeOf<
+      FunctionTool<Record<string, never>, { userId: number }, number>
+    >();
+  });
+});