@livekit/agents 1.0.17 → 1.0.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
@@ -39,21 +39,28 @@
   "dependencies": {
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "@livekit/mutex": "^1.1.1",
-    "@livekit/protocol": "^1.41.0",
+    "@livekit/protocol": "^1.43.0",
     "@livekit/typed-emitter": "^3.0.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.54.0",
+    "@opentelemetry/otlp-exporter-base": "^0.208.0",
+    "@opentelemetry/resources": "^1.28.0",
+    "@opentelemetry/sdk-trace-base": "^1.28.0",
+    "@opentelemetry/sdk-trace-node": "^1.28.0",
+    "@opentelemetry/semantic-conventions": "^1.28.0",
     "@types/pidusage": "^2.0.5",
     "commander": "^12.0.0",
     "fluent-ffmpeg": "^2.1.3",
     "heap-js": "^2.6.0",
     "json-schema": "^0.4.0",
-    "livekit-server-sdk": "^2.13.3",
-    "openai": "^4.91.1",
+    "livekit-server-sdk": "^2.14.1",
+    "openai": "^6.8.1",
     "pidusage": "^4.0.1",
     "pino": "^8.19.0",
     "pino-pretty": "^11.0.0",
     "sharp": "0.34.3",
     "uuid": "^11.1.0",
-    "ws": "^8.16.0",
+    "ws": "^8.18.0",
     "zod-to-json-schema": "^3.24.6"
   },
   "peerDependencies": {

package/src/index.ts

@@ -16,6 +16,7 @@ import * as llm from './llm/index.js';
 import * as metrics from './metrics/index.js';
 import * as stream from './stream/index.js';
 import * as stt from './stt/index.js';
+import * as telemetry from './telemetry/index.js';
 import * as tokenize from './tokenize/index.js';
 import * as tts from './tts/index.js';
 import * as voice from './voice/index.js';
@@ -34,4 +35,4 @@ export * from './vad.js';
 export * from './version.js';
 export * from './worker.js';
 
-export { cli, inference, ipc, llm, metrics, stream, stt, tokenize, tts, voice };
+export { cli, inference, ipc, llm, metrics, stream, stt, telemetry, tokenize, tts, voice };

package/src/inference/llm.ts

@@ -7,6 +7,7 @@ import {
   APIStatusError,
   APITimeoutError,
   DEFAULT_API_CONNECT_OPTIONS,
+  type Expand,
   toError,
 } from '../index.js';
 import * as llm from '../llm/index.js';
@@ -34,9 +35,10 @@ export type KimiModels = 'moonshotai/kimi-k2-instruct';
 
 export type DeepSeekModels = 'deepseek-ai/deepseek-v3';
 
-type ChatCompletionPredictionContentParam = OpenAI.Chat.Completions.ChatCompletionPredictionContent;
-type WebSearchOptions = OpenAI.Chat.Completions.ChatCompletionCreateParams.WebSearchOptions;
-type ToolChoice = OpenAI.Chat.Completions.ChatCompletionCreateParams['tool_choice'];
+type ChatCompletionPredictionContentParam =
+  Expand<OpenAI.Chat.Completions.ChatCompletionPredictionContent>;
+type WebSearchOptions = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams.WebSearchOptions>;
+type ToolChoice = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams['tool_choice']>;
 type Verbosity = 'low' | 'medium' | 'high';
 
 export interface ChatCompletionOptions extends Record<string, unknown> {
@@ -86,6 +88,7 @@ export interface InferenceLLMOptions {
   apiKey: string;
   apiSecret: string;
   modelOptions: ChatCompletionOptions;
+  strictToolSchema?: boolean;
 }
 
 export interface GatewayOptions {
@@ -107,10 +110,19 @@ export class LLM extends llm.LLM {
     apiKey?: string;
     apiSecret?: string;
     modelOptions?: InferenceLLMOptions['modelOptions'];
+    strictToolSchema?: boolean;
   }) {
     super();
 
-    const { model, provider, baseURL, apiKey, apiSecret, modelOptions } = opts;
+    const {
+      model,
+      provider,
+      baseURL,
+      apiKey,
+      apiSecret,
+      modelOptions,
+      strictToolSchema = false,
+    } = opts;
 
     const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
@@ -131,6 +143,7 @@ export class LLM extends llm.LLM {
       apiKey: lkApiKey,
       apiSecret: lkApiSecret,
       modelOptions: modelOptions || {},
+      strictToolSchema,
     };
 
     this.client = new OpenAI({
@@ -180,9 +193,13 @@ export class LLM extends llm.LLM {
       modelOptions.parallel_tool_calls = parallelToolCalls;
     }
 
-    toolChoice = toolChoice !== undefined ? toolChoice : this.opts.modelOptions.tool_choice;
+    toolChoice =
+      toolChoice !== undefined
+        ? toolChoice
+        : (this.opts.modelOptions.tool_choice as llm.ToolChoice | undefined);
+
     if (toolChoice) {
-      modelOptions.tool_choice = toolChoice;
+      modelOptions.tool_choice = toolChoice as ToolChoice;
     }
 
     // TODO(AJS-270): Add response_format support here
@@ -197,6 +214,7 @@ export class LLM extends llm.LLM {
       toolCtx,
       connOptions,
       modelOptions,
+      strictToolSchema: this.opts.strictToolSchema ?? false, // default to false if not set
       gatewayOptions: {
         apiKey: this.opts.apiKey,
         apiSecret: this.opts.apiSecret,
@@ -211,6 +229,7 @@ export class LLMStream extends llm.LLMStream {
   private providerFmt: llm.ProviderFormat;
   private client: OpenAI;
   private modelOptions: Record<string, unknown>;
+  private strictToolSchema: boolean;
 
   private gatewayOptions?: GatewayOptions;
   private toolCallId?: string;
@@ -230,6 +249,7 @@ export class LLMStream extends llm.LLMStream {
       connOptions,
       modelOptions,
       providerFmt,
+      strictToolSchema,
     }: {
       model: LLMModels;
       provider?: string;
@@ -238,8 +258,9 @@ export class LLMStream extends llm.LLMStream {
       toolCtx?: llm.ToolContext;
       gatewayOptions?: GatewayOptions;
       connOptions: APIConnectOptions;
-      modelOptions: Record<string, any>;
+      modelOptions: Record<string, unknown>;
       providerFmt?: llm.ProviderFormat;
+      strictToolSchema: boolean;
     },
   ) {
     super(llm, { chatCtx, toolCtx, connOptions });
@@ -249,6 +270,7 @@ export class LLMStream extends llm.LLMStream {
     this.providerFmt = providerFmt || 'openai';
     this.modelOptions = modelOptions;
     this.model = model;
+    this.strictToolSchema = strictToolSchema;
   }
 
   protected async run(): Promise<void> {
@@ -263,16 +285,26 @@ export class LLMStream extends llm.LLMStream {
       )) as OpenAI.ChatCompletionMessageParam[];
 
       const tools = this.toolCtx
-        ? Object.entries(this.toolCtx).map(([name, func]) => ({
-            type: 'function' as const,
-            function: {
-              name,
-              description: func.description,
-              parameters: llm.toJsonSchema(
-                func.parameters,
-              ) as unknown as OpenAI.Chat.Completions.ChatCompletionTool['function']['parameters'],
-            },
-          }))
+        ? Object.entries(this.toolCtx).map(([name, func]) => {
+            const oaiParams = {
+              type: 'function' as const,
+              function: {
+                name,
+                description: func.description,
+                parameters: llm.toJsonSchema(
+                  func.parameters,
+                  true,
+                  this.strictToolSchema,
+                ) as unknown as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function']['parameters'],
+              } as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function'],
+            };
+
+            if (this.strictToolSchema) {
+              oaiParams.function.strict = true;
+            }
+
+            return oaiParams;
+          })
         : undefined;
 
       const requestOptions: Record<string, unknown> = { ...this.modelOptions };
@@ -345,7 +377,7 @@ export class LLMStream extends llm.LLMStream {
           options: {
             statusCode: error.status,
             body: error.error,
-            requestId: error.request_id,
+            requestId: error.requestID,
             retryable,
           },
         });
@@ -387,10 +419,10 @@ export class LLMStream extends llm.LLMStream {
          *
          * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)
          * [ChoiceDeltaToolCall(index=0, id='call_LaVeHWUHpef9K1sd5UO8TtLg', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]
-         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='{"location": "P', name=None), type=None)]
-         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris}', name=None), type=None)]
+         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='\{"location": "P', name=None), type=None)]
+         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris\}', name=None), type=None)]
          * [ChoiceDeltaToolCall(index=1, id='call_ThU4OmMdQXnnVmpXGOCknXIB', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]
-         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='{"location": "T', name=None), type=None)]
+         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='\{"location": "T', name=None), type=None)]
          * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='okyo', name=None), type=None)]
          * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=None), finish_reason='tool_calls', index=0, logprobs=None)
          */

package/src/inference/tts.ts

@@ -421,7 +421,7 @@ export class SynthesizeStream<TModel extends TTSModels> extends BaseSynthesizeSt
         createRecvTask(),
       ]);
     } catch (e) {
-      this.#logger.error('Error in SynthesizeStream', { error: e });
+      this.#logger.error({ error: e }, 'Error in SynthesizeStream');
     } finally {
       resourceCleanup();
     }

package/src/ipc/job_proc_lazy_main.ts

@@ -88,7 +88,7 @@ const startJob = (
 
   const ctx = new JobContext(proc, info, room, onConnect, onShutdown, new InfClient());
 
-  const task = new Promise<void>(async () => {
+  const task = (async () => {
     const unconnectedTimeout = setTimeout(() => {
       if (!(connect || shutdown)) {
         logger.warn(
@@ -109,6 +109,14 @@ const startJob = (
       process.send!({ case: 'exiting', value: { reason: close[1] } });
     });
 
+    // Close the primary agent session if it exists
+    if (ctx._primaryAgentSession) {
+      await ctx._primaryAgentSession.close();
+    }
+
+    // Generate and save/upload session report
+    await ctx._onSessionEnd();
+
     await room.disconnect();
     logger.debug('disconnected from room');
 
@@ -122,7 +130,7 @@ const startJob = (
 
     process.send!({ case: 'done' });
     joinFuture.resolve();
-  });
+  })();
 
   return { ctx, task };
 };

package/src/job.ts

@@ -14,6 +14,8 @@ import { AsyncLocalStorage } from 'node:async_hooks';
 import type { Logger } from 'pino';
 import type { InferenceExecutor } from './ipc/inference_executor.js';
 import { log } from './log.js';
+import type { AgentSession } from './voice/agent_session.js';
+import { type SessionReport, createSessionReport } from './voice/report.js';
 
 // AsyncLocalStorage for job context, similar to Python's contextvars
 const jobContextStorage = new AsyncLocalStorage<JobContext>();
@@ -79,6 +81,8 @@ export class FunctionExistsError extends Error {
 }
 
 /** The job and environment context as seen by the agent, accessible by the entrypoint function. */
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('job_entrypoint') wrapper in entrypoint
+// TODO(brian): PR5 - Add uploadSessionReport() call in cleanup/session end
 export class JobContext {
   #proc: JobProcess;
   #info: RunningJobInfo;
@@ -97,6 +101,9 @@ export class JobContext {
   #logger: Logger;
   #inferenceExecutor: InferenceExecutor;
 
+  /** @internal */
+  _primaryAgentSession?: AgentSession;
+
   private connected: boolean = false;
 
   constructor(
@@ -232,6 +239,47 @@ export class JobContext {
     this.connected = true;
   }
 
+  makeSessionReport(session?: AgentSession): SessionReport {
+    const targetSession = session || this._primaryAgentSession;
+
+    if (!targetSession) {
+      throw new Error('Cannot prepare report, no AgentSession was found');
+    }
+
+    // TODO(brian): implement and check recorder io
+    // TODO(brian): PR5 - Ensure chat history serialization includes all required fields (use sessionReportToJSON helper)
+
+    return createSessionReport({
+      jobId: this.job.id,
+      roomId: this.job.room?.sid || '',
+      room: this.job.room?.name || '',
+      options: targetSession.options,
+      events: targetSession._recordedEvents,
+      enableUserDataTraining: true,
+      chatHistory: targetSession.history.copy(),
+    });
+  }
+
+  async _onSessionEnd(): Promise<void> {
+    const session = this._primaryAgentSession;
+    if (!session) {
+      return;
+    }
+
+    const report = this.makeSessionReport(session);
+
+    // TODO(brian): Implement CLI/console
+
+    // TODO(brian): PR5 - Call uploadSessionReport() if report.enableUserDataTraining is true
+    // TODO(brian): PR5 - Upload includes: multipart form with header (protobuf), chat_history (JSON), and audio recording (if available)
+
+    this.#logger.debug('Session ended, report generated', {
+      jobId: report.jobId,
+      roomId: report.roomId,
+      eventsCount: report.events.length,
+    });
+  }
+
   /**
    * Gracefully shuts down the job, and runs all shutdown promises.
    *

package/src/llm/__snapshots__/zod-utils.test.ts.snap

@@ -339,3 +339,221 @@ exports[`Zod Utils > zodSchemaToJsonSchema > Zod v4 schemas > should handle v4 s
   "type": "object",
 }
 `;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle arrays in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "numbers": {
+      "items": {
+        "type": "number",
+      },
+      "type": "array",
+    },
+    "tags": {
+      "items": {
+        "type": "string",
+      },
+      "type": "array",
+    },
+  },
+  "required": [
+    "tags",
+    "numbers",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle default values in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "active": {
+      "default": true,
+      "type": "boolean",
+    },
+    "name": {
+      "type": "string",
+    },
+    "role": {
+      "default": "user",
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "role",
+    "active",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle nested objects in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "metadata": {
+      "additionalProperties": false,
+      "properties": {
+        "created": {
+          "type": "string",
+        },
+      },
+      "required": [
+        "created",
+      ],
+      "type": "object",
+    },
+    "user": {
+      "additionalProperties": false,
+      "properties": {
+        "email": {
+          "anyOf": [
+            {
+              "type": "string",
+            },
+            {
+              "type": "null",
+            },
+          ],
+        },
+        "name": {
+          "type": "string",
+        },
+      },
+      "required": [
+        "name",
+        "email",
+      ],
+      "type": "object",
+    },
+  },
+  "required": [
+    "user",
+    "metadata",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle nullable fields in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "optional": {
+      "anyOf": [
+        {
+          "type": "string",
+        },
+        {
+          "type": "null",
+        },
+      ],
+    },
+    "required": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "required",
+    "optional",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle optional fields in strict mode 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "optional": {
+      "anyOf": [
+        {
+          "type": "string",
+        },
+        {
+          "type": "null",
+        },
+      ],
+    },
+    "required": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "required",
+    "optional",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should handle v3 schemas in strict mode 1`] = `
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "age": {
+      "type": [
+        "number",
+        "null",
+      ],
+    },
+    "name": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "age",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should produce standard JSON schema with strict: false 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "age": {
+      "type": "number",
+    },
+    "name": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "age",
+  ],
+  "type": "object",
+}
+`;
+
+exports[`Zod Utils > zodSchemaToJsonSchema > strict parameter > should produce strict JSON schema with strict: true 1`] = `
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "properties": {
+    "age": {
+      "type": "number",
+    },
+    "name": {
+      "type": "string",
+    },
+  },
+  "required": [
+    "name",
+    "age",
+  ],
+  "type": "object",
+}
+`;

package/src/llm/chat_context.ts

@@ -300,7 +300,59 @@ export class FunctionCallOutput {
   }
 }
 
-export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput;
+export class AgentHandoffItem {
+  readonly id: string;
+
+  readonly type = 'agent_handoff' as const;
+
+  oldAgentId: string | undefined;
+
+  newAgentId: string;
+
+  createdAt: number;
+
+  constructor(params: {
+    oldAgentId?: string;
+    newAgentId: string;
+    id?: string;
+    createdAt?: number;
+  }) {
+    const { oldAgentId, newAgentId, id = shortuuid('item_'), createdAt = Date.now() } = params;
+    this.id = id;
+    this.oldAgentId = oldAgentId;
+    this.newAgentId = newAgentId;
+    this.createdAt = createdAt;
+  }
+
+  static create(params: {
+    oldAgentId?: string;
+    newAgentId: string;
+    id?: string;
+    createdAt?: number;
+  }) {
+    return new AgentHandoffItem(params);
+  }
+
+  toJSON(excludeTimestamp: boolean = false): JSONValue {
+    const result: JSONValue = {
+      id: this.id,
+      type: this.type,
+      newAgentId: this.newAgentId,
+    };
+
+    if (this.oldAgentId !== undefined) {
+      result.oldAgentId = this.oldAgentId;
+    }
+
+    if (!excludeTimestamp) {
+      result.createdAt = this.createdAt;
+    }
+
+    return result;
+  }
+}
+
+export type ChatItem = ChatMessage | FunctionCall | FunctionCallOutput | AgentHandoffItem;
 
 export class ChatContext {
   protected _items: ChatItem[];

package/src/llm/index.ts

@@ -17,6 +17,7 @@ export {
 } from './tool_context.js';
 
 export {
+  AgentHandoffItem,
   ChatContext,
   ChatMessage,
   createAudioContent,

package/src/llm/llm.ts

@@ -78,7 +78,7 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCal
     connOptions?: APIConnectOptions;
     parallelToolCalls?: boolean;
     toolChoice?: ToolChoice;
-    extraKwargs?: Record<string, any>;
+    extraKwargs?: Record<string, unknown>;
   }): LLMStream;
 
   /**
@@ -136,8 +136,10 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   }
 
   private async mainTask() {
+    // TODO(brian): PR3 - Add span wrapping: tracer.startActiveSpan('llm_request', ..., { endOnExit: false })
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
+        // TODO(brian): PR3 - Add span for retry attempts: tracer.startActiveSpan('llm_request_run', ...)
         return await this.run();
       } catch (error) {
         if (error instanceof APIError) {