@livekit/agents 0.3.4 → 0.4.0

package/dist/llm/function_context.js

@@ -5,33 +5,69 @@ import { z } from 'zod';
 /** @internal */
 export const oaiParams = (p) => {
     const properties = {};
-    const required_properties = [];
-    for (const key in p.shape) {
-        const field = p.shape[key];
-        const description = field._def.description || undefined;
-        let type;
-        let enumValues;
-        if (field instanceof z.ZodEnum) {
-            enumValues = field._def.values;
-            type = typeof enumValues[0];
+    const requiredProperties = [];
+    const processZodType = (field) => {
+        const isOptional = field instanceof z.ZodOptional;
+        const nestedField = isOptional ? field._def.innerType : field;
+        const description = field._def.description;
+        if (nestedField instanceof z.ZodEnum) {
+            return {
+                type: typeof nestedField._def.values[0],
+                ...(description && { description }),
+                enum: nestedField._def.values,
+            };
+        }
+        else if (nestedField instanceof z.ZodArray) {
+            const elementType = nestedField._def.type;
+            return {
+                type: 'array',
+                ...(description && { description }),
+                items: processZodType(elementType),
+            };
+        }
+        else if (nestedField instanceof z.ZodObject) {
+            const { properties, required } = oaiParams(nestedField);
+            return {
+                type: 'object',
+                ...(description && { description }),
+                properties,
+                required,
+            };
         }
         else {
-            type = field._def.typeName.toLowerCase();
+            let type = nestedField._def.typeName.toLowerCase();
+            type = type.includes('zod') ? type.substring(3) : type;
+            return {
+                type,
+                ...(description && { description }),
+            };
         }
-        properties[key] = {
-            type: type.includes('zod') ? type.substring(3) : type,
-            description,
-            enum: enumValues,
-        };
-        if (!field._def.defaultValue) {
-            required_properties.push(key);
+    };
+    for (const key in p.shape) {
+        const field = p.shape[key];
+        properties[key] = processZodType(field);
+        if (!(field instanceof z.ZodOptional)) {
+            requiredProperties.push(key);
         }
     }
     const type = 'object';
     return {
         type,
         properties,
-        required_properties,
+        required: requiredProperties,
+    };
+};
+/** @internal */
+export const oaiBuildFunctionInfo = (fncCtx, toolCallId, fncName, rawArgs) => {
+    if (!fncCtx[fncName]) {
+        throw new Error(`AI function ${fncName} not found`);
+    }
+    return {
+        name: fncName,
+        func: fncCtx[fncName],
+        toolCallId,
+        rawParams: rawArgs,
+        params: JSON.parse(rawArgs),
     };
 };
 //# sourceMappingURL=function_context.js.map

package/dist/llm/function_context.js.map

@@ -1 +1 @@
-{"version":3,"file":"function_context.js","sourceRoot":"","sources":["../../src/llm/function_context.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAsBxB,gBAAgB;AAChB,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAiB,EAAE,EAAE;IAC7C,MAAM,UAAU,GAAwB,EAAE,CAAC;IAC3C,MAAM,mBAAmB,GAAa,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3B,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC;QACxD,IAAI,IAAY,CAAC;QACjB,IAAI,UAA6B,CAAC;QAElC,IAAI,KAAK,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;YAC/B,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YAC/B,IAAI,GAAG,OAAO,UAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC3C,CAAC;QAED,UAAU,CAAC,GAAG,CAAC,GAAG;YAChB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YACrD,WAAW;YACX,IAAI,EAAE,UAAU;SACjB,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,QAAiB,CAAC;IAC/B,OAAO;QACL,IAAI;QACJ,UAAU;QACV,mBAAmB;KACpB,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"function_context.js","sourceRoot":"","sources":["../../src/llm/function_context.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAwCxB,gBAAgB;AAChB,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAiB,EAAE,EAAE;IAC7C,MAAM,UAAU,GAAwB,EAAE,CAAC;IAC3C,MAAM,kBAAkB,GAAa,EAAE,CAAC;IAExC,MAAM,cAAc,GAAG,CAAC,KAAmB,EAAO,EAAE;QAClD,MAAM,UAAU,GAAG,KAAK,YAAY,CAAC,CAAC,WAAW,CAAC;QAClD,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QAC9D,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC;QAE3C,IAAI,WAAW,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;YACrC,OAAO;gBACL,IAAI,EAAE,OAAO,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBACvC,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;gBACnC,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM;aAC9B,CAAC;QACJ,CAAC;aAAM,IAAI,WAAW,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC7C,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1C,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;gBACnC,KAAK,EAAE,cAAc,CAAC,WAAW,CAAC;aACnC,CAAC;QACJ,CAAC;aAAM,IAAI,WAAW,YAAY,CAAC,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YACxD,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;gBACnC,UAAU;gBACV,QAAQ;aACT,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACnD,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACvD,OAAO;gBACL,IAAI;gBACJ,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;aACpC,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3B,UAAU,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAExC,IAAI,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,QAAiB,CAAC;IAC/B,OAAO;QACL,IAAI;QACJ,UAAU;QACV,QAAQ,EAAE,kBAAkB;KAC7B,CAAC;AACJ,CAAC,CAAC;AAEF,gBAAgB;AAChB,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,MAAuB,EACvB,UAAkB,EAClB,OAAe,EACf,OAAe,EACG,EAAE;IACpB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,eAAe,OAAO,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACL,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC;QACrB,UAAU;QACV,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;KAC5B,CAAC;AACJ,CAAC,CAAC"}

package/dist/llm/function_context.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=function_context.test.d.ts.map

package/dist/llm/function_context.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"function_context.test.d.ts","sourceRoot":"","sources":["../../src/llm/function_context.test.ts"],"names":[],"mappings":""}

package/dist/llm/function_context.test.js

@@ -0,0 +1,218 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { z } from 'zod';
+import { oaiParams } from './function_context.js';
+describe('function_context', () => {
+    describe('oaiParams', () => {
+        it('should handle basic object schema', () => {
+            const schema = z.object({
+                name: z.string().describe('The user name'),
+                age: z.number().describe('The user age'),
+            });
+            const result = oaiParams(schema);
+            expect(result).toEqual({
+                type: 'object',
+                properties: {
+                    name: {
+                        type: 'string',
+                        description: 'The user name',
+                    },
+                    age: {
+                        type: 'number',
+                        description: 'The user age',
+                    },
+                },
+                required: ['name', 'age'],
+            });
+        });
+        it('should handle enum fields', () => {
+            const schema = z.object({
+                color: z.enum(['red', 'blue', 'green']).describe('Choose a color'),
+            });
+            const result = oaiParams(schema);
+            expect(result).toEqual({
+                type: 'object',
+                properties: {
+                    color: {
+                        type: 'string',
+                        description: 'Choose a color',
+                        enum: ['red', 'blue', 'green'],
+                    },
+                },
+                required: ['color'],
+            });
+        });
+        it('should handle array fields', () => {
+            const schema = z.object({
+                tags: z.array(z.string()).describe('List of tags'),
+            });
+            const result = oaiParams(schema);
+            expect(result).toEqual({
+                type: 'object',
+                properties: {
+                    tags: {
+                        type: 'array',
+                        description: 'List of tags',
+                        items: {
+                            type: 'string',
+                        },
+                    },
+                },
+                required: ['tags'],
+            });
+        });
+        it('should handle array of enums', () => {
+            const schema = z.object({
+                colors: z.array(z.enum(['red', 'blue', 'green'])).describe('List of colors'),
+            });
+            const result = oaiParams(schema);
+            expect(result).toEqual({
+                type: 'object',
+                properties: {
+                    colors: {
+                        type: 'array',
+                        description: 'List of colors',
+                        items: {
+                            type: 'string',
+                            enum: ['red', 'blue', 'green'],
+                        },
+                    },
+                },
+                required: ['colors'],
+            });
+        });
+        it('should handle optional fields', () => {
+            const schema = z.object({
+                name: z.string().describe('The user name'),
+                age: z.number().optional().describe('The user age'),
+            });
+            const result = oaiParams(schema);
+            expect(result).toEqual({
+                type: 'object',
+                properties: {
+                    name: {
+                        type: 'string',
+                        description: 'The user name',
+                    },
+                    age: {
+                        type: 'number',
+                        description: 'The user age',
+                    },
+                },
+                required: ['name'], // age should not be required
+            });
+        });
+        it('should handle fields without descriptions', () => {
+            const schema = z.object({
+                name: z.string(),
+                age: z.number(),
+            });
+            const result = oaiParams(schema);
+            expect(result).toEqual({
+                type: 'object',
+                properties: {
+                    name: {
+                        type: 'string',
+                        description: undefined,
+                    },
+                    age: {
+                        type: 'number',
+                        description: undefined,
+                    },
+                },
+                required: ['name', 'age'],
+            });
+        });
+    });
+    describe('CallableFunction type', () => {
+        it('should properly type a callable function', async () => {
+            const schema = z.object({
+                name: z.string().describe('The user name'),
+                age: z.number().describe('The user age'),
+            });
+            const testFunction = {
+                description: 'Test function',
+                parameters: schema,
+                execute: async (args) => {
+                    // TypeScript should recognize args.name and args.age
+                    return `${args.name} is ${args.age} years old`;
+                },
+            };
+            const result = await testFunction.execute({ name: 'John', age: 30 });
+            expect(result).toBe('John is 30 years old');
+        });
+        it('should handle async execution', async () => {
+            const schema = z.object({
+                delay: z.number().describe('Delay in milliseconds'),
+            });
+            const testFunction = {
+                description: 'Async test function',
+                parameters: schema,
+                execute: async (args) => {
+                    await new Promise((resolve) => setTimeout(resolve, args.delay));
+                    return args.delay;
+                },
+            };
+            const start = Date.now();
+            const result = await testFunction.execute({ delay: 100 });
+            const duration = Date.now() - start;
+            expect(result).toBe(100);
+            expect(duration).toBeGreaterThanOrEqual(95); // Allow for small timing variations
+        });
+        describe('nested array support', () => {
+            it('should handle nested array fields', () => {
+                const schema = z.object({
+                    items: z.array(z.object({
+                        name: z.string().describe('the item name'),
+                        modifiers: z
+                            .array(z.object({
+                            modifier_name: z.string(),
+                            modifier_value: z.string(),
+                        }))
+                            .describe('list of the modifiers applied on this item, such as size'),
+                    })),
+                });
+                const result = oaiParams(schema);
+                expect(result).toEqual({
+                    type: 'object',
+                    properties: {
+                        items: {
+                            type: 'array',
+                            description: undefined,
+                            items: {
+                                type: 'object',
+                                properties: {
+                                    name: {
+                                        type: 'string',
+                                        description: 'the item name',
+                                    },
+                                    modifiers: {
+                                        type: 'array',
+                                        description: 'list of the modifiers applied on this item, such as size',
+                                        items: {
+                                            type: 'object',
+                                            properties: {
+                                                modifier_name: {
+                                                    type: 'string',
+                                                },
+                                                modifier_value: {
+                                                    type: 'string',
+                                                },
+                                            },
+                                            required: ['modifier_name', 'modifier_value'],
+                                        },
+                                    },
+                                },
+                                required: ['name', 'modifiers'],
+                            },
+                        },
+                    },
+                    required: ['items'],
+                });
+            });
+        });
+    });
+});
+//# sourceMappingURL=function_context.test.js.map

package/dist/llm/function_context.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"function_context.test.js","sourceRoot":"","sources":["../../src/llm/function_context.test.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAElD,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAC1C,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;aACzC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,eAAe;qBAC7B;oBACD,GAAG,EAAE;wBACH,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,cAAc;qBAC5B;iBACF;gBACD,QAAQ,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;aAC1B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aACnE,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE;wBACL,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,gBAAgB;wBAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC;qBAC/B;iBACF;gBACD,QAAQ,EAAE,CAAC,OAAO,CAAC;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC;aACnD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE;wBACJ,IAAI,EAAE,OAAO;wBACb,WAAW,EAAE,cAAc;wBAC3B,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF;gBACD,QAAQ,EAAE,CAAC,MAAM,CAAC;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7E,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,MAAM,EAAE;wBACN,IAAI,EAAE,OAAO;wBACb,WAAW,EAAE,gBAAgB;wBAC7B,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC;yBAC/B;qBACF;iBACF;gBACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;aACrB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAC1C,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;aACpD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,eAAe;qBAC7B;oBACD,GAAG,EAAE;wBACH,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,cAAc;qBAC5B;iBACF;gBACD,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,6BAA6B;aAClD,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;gBAChB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;aAChB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAEjC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,SAAS;qBACvB;oBACD,GAAG,EAAE;wBACH,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,SAAS;qBACvB;iBACF;gBACD,QAAQ,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;aAC1B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAC1C,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;aACzC,CAAC,CAAC;YAEH,MAAM,YAAY,GAA4C;gBAC5D,WAAW,EAAE,eAAe;gBAC5B,UAAU,EAAE,MAAM;gBAClB,OAAO,EAAE,KAAK,EAAE,IAA4B,EAAE,EAAE;oBAC9C,qDAAqD;oBACrD,OAAO,GAAG,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,GAAG,YAAY,CAAC;gBACjD,CAAC;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;gBACtB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;aACpD,CAAC,CAAC;YAEH,MAAM,YAAY,GAA4C;gBAC5D,WAAW,EAAE,qBAAqB;gBAClC,UAAU,EAAE,MAAM;gBAClB,OAAO,EAAE,KAAK,EAAE,IAA4B,EAAE,EAAE;oBAC9C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAChE,OAAO,IAAI,CAAC,KAAK,CAAC;gBACpB,CAAC;aACF,CAAC;YAEF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YAEpC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,MAAM,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,oCAAoC;QACnF,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;YACpC,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;gBAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;oBACtB,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,MAAM,CAAC;wBACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;wBAC1C,SAAS,EAAE,CAAC;6BACT,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;4BACP,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;4BACzB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;yBAC3B,CAAC,CACH;6BACA,QAAQ,CAAC,0DAA0D,CAAC;qBACxE,CAAC,CACH;iBACF,CAAC,CAAC;gBACH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;gBACjC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;oBACrB,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,KAAK,EAAE;4BACL,IAAI,EAAE,OAAO;4BACb,WAAW,EAAE,SAAS;4BACtB,KAAK,EAAE;gCACL,IAAI,EAAE,QAAQ;gCACd,UAAU,EAAE;oCACV,IAAI,EAAE;wCACJ,IAAI,EAAE,QAAQ;wCACd,WAAW,EAAE,eAAe;qCAC7B;oCACD,SAAS,EAAE;wCACT,IAAI,EAAE,OAAO;wCACb,WAAW,EAAE,0DAA0D;wCACvE,KAAK,EAAE;4CACL,IAAI,EAAE,QAAQ;4CACd,UAAU,EAAE;gDACV,aAAa,EAAE;oDACb,IAAI,EAAE,QAAQ;iDACf;gDACD,cAAc,EAAE;oDACd,IAAI,EAAE,QAAQ;iDACf;6CACF;4CACD,QAAQ,EAAE,CAAC,eAAe,EAAE,gBAAgB,CAAC;yCAC9C;qCACF;iCACF;gCACD,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC;6BAChC;yBACF;qBACF;oBACD,QAAQ,EAAE,CAAC,OAAO,CAAC;iBACpB,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/llm/index.d.ts

@@ -1,3 +1,4 @@
-import { type CallableFunction, type FunctionContext, type inferParameters, oaiParams } from './function_context.js';
-export { CallableFunction, FunctionContext, inferParameters, oaiParams };
+export { type CallableFunction, type FunctionCallInfo, type CallableFunctionResult, type FunctionContext, type inferParameters, oaiParams, oaiBuildFunctionInfo, } from './function_context.js';
+export { type ChatImage, type ChatAudio, type ChatContent, ChatRole, ChatMessage, ChatContext, } from './chat_context.js';
+export { type ChoiceDelta, type CompletionUsage, type Choice, type ChatChunk, LLM, LLMStream, } from './llm.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,SAAS,EACV,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,EAC3B,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,SAAS,EACT,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EACL,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,QAAQ,EACR,WAAW,EACX,WAAW,GACZ,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,MAAM,EACX,KAAK,SAAS,EACd,GAAG,EACH,SAAS,GACV,MAAM,UAAU,CAAC"}

package/dist/llm/index.js

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { oaiParams, } from './function_context.js';
-export { oaiParams };
+export { oaiParams, oaiBuildFunctionInfo, } from './function_context.js';
+export { ChatRole, ChatMessage, ChatContext, } from './chat_context.js';
+export { LLM, LLMStream, } from './llm.js';
 //# sourceMappingURL=index.js.map

package/dist/llm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAIL,SAAS,GACV,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAsD,SAAS,EAAE,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAML,SAAS,EACT,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAIL,QAAQ,EACR,WAAW,EACX,WAAW,GACZ,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAKL,GAAG,EACH,SAAS,GACV,MAAM,UAAU,CAAC"}

package/dist/llm/llm.d.ts

@@ -0,0 +1,53 @@
+import { AsyncIterableQueue } from '../utils.js';
+import type { ChatContext, ChatRole } from './chat_context.js';
+import type { FunctionCallInfo, FunctionContext } from './function_context.js';
+export interface ChoiceDelta {
+    role: ChatRole;
+    content?: string;
+    toolCalls?: FunctionCallInfo[];
+}
+export interface CompletionUsage {
+    completionTokens: number;
+    promptTokens: number;
+    totalTokens: number;
+}
+export interface Choice {
+    delta: ChoiceDelta;
+    index: number;
+}
+export interface ChatChunk {
+    requestId: string;
+    choices: Choice[];
+    usage?: CompletionUsage;
+}
+export declare abstract class LLM {
+    /**
+     * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
+     */
+    abstract chat({ chatCtx, fncCtx, temperature, n, parallelToolCalls, }: {
+        chatCtx: ChatContext;
+        fncCtx?: FunctionContext;
+        temperature?: number;
+        n?: number;
+        parallelToolCalls?: boolean;
+    }): LLMStream;
+}
+export declare abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+    #private;
+    protected queue: AsyncIterableQueue<ChatChunk>;
+    protected closed: boolean;
+    protected _functionCalls: FunctionCallInfo[];
+    constructor(chatCtx: ChatContext, fncCtx?: FunctionContext);
+    /** List of called functions from this stream. */
+    get functionCalls(): FunctionCallInfo[];
+    /** The function context of this stream. */
+    get fncCtx(): FunctionContext | undefined;
+    /** The initial chat context of this stream. */
+    get chatCtx(): ChatContext;
+    /** Execute all deferred functions of this stream concurrently. */
+    executeFunctions(): FunctionCallInfo[];
+    next(): Promise<IteratorResult<ChatChunk>>;
+    close(): void;
+    [Symbol.asyncIterator](): LLMStream;
+}
+//# sourceMappingURL=llm.d.ts.map

package/dist/llm/llm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAE/E,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,8BAAsB,GAAG;IACvB;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,MAAM,EACN,WAAW,EACX,CAAC,EACD,iBAAiB,GAClB,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,iBAAiB,CAAC,EAAE,OAAO,CAAC;KAC7B,GAAG,SAAS;CACd;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,cAAc,EAAE,gBAAgB,EAAE,CAAM;gBAKtC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,eAAe;IAK1D,iDAAiD;IACjD,IAAI,aAAa,IAAI,gBAAgB,EAAE,CAEtC;IAED,2CAA2C;IAC3C,IAAI,MAAM,IAAI,eAAe,GAAG,SAAS,CAExC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,kEAAkE;IAClE,gBAAgB,IAAI,gBAAgB,EAAE;IAWtC,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAKL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}

package/dist/llm/llm.js

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { AsyncIterableQueue } from '../utils.js';
+export class LLM {
+}
+export class LLMStream {
+    queue = new AsyncIterableQueue();
+    closed = false;
+    _functionCalls = [];
+    #chatCtx;
+    #fncCtx;
+    constructor(chatCtx, fncCtx) {
+        this.#chatCtx = chatCtx;
+        this.#fncCtx = fncCtx;
+    }
+    /** List of called functions from this stream. */
+    get functionCalls() {
+        return this._functionCalls;
+    }
+    /** The function context of this stream. */
+    get fncCtx() {
+        return this.#fncCtx;
+    }
+    /** The initial chat context of this stream. */
+    get chatCtx() {
+        return this.#chatCtx;
+    }
+    /** Execute all deferred functions of this stream concurrently. */
+    executeFunctions() {
+        this._functionCalls.forEach((f) => (f.task = f.func.execute(f.params).then((result) => ({ name: f.name, toolCallId: f.toolCallId, result }), (error) => ({ name: f.name, toolCallId: f.toolCallId, error }))));
+        return this._functionCalls;
+    }
+    next() {
+        return this.queue.next();
+    }
+    close() {
+        this.queue.close();
+        this.closed = true;
+    }
+    [Symbol.asyncIterator]() {
+        return this;
+    }
+}
+//# sourceMappingURL=llm.js.map

package/dist/llm/llm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AA2BjD,MAAM,OAAgB,GAAG;CAiBxB;AAED,MAAM,OAAgB,SAAS;IACnB,KAAK,GAAG,IAAI,kBAAkB,EAAa,CAAC;IAC5C,MAAM,GAAG,KAAK,CAAC;IACf,cAAc,GAAuB,EAAE,CAAC;IAElD,QAAQ,CAAc;IACtB,OAAO,CAAmB;IAE1B,YAAY,OAAoB,EAAE,MAAwB;QACxD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,iDAAiD;IACjD,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,2CAA2C;IAC3C,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,+CAA+C;IAC/C,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,kEAAkE;IAClE,gBAAgB;QACd,IAAI,CAAC,cAAc,CAAC,OAAO,CACzB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CACrC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,EAChE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,CAC/D,CAAC,CACL,CAAC;QACF,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,aAAa,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/multimodal/agent_playout.d.ts

@@ -1,4 +1,4 @@
-/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" />
 import type { AudioFrame } from '@livekit/rtc-node';
 import { type AudioSource } from '@livekit/rtc-node';
 import { EventEmitter } from 'node:events';