@livekit/agents 1.1.0 → 1.2.0

package/src/llm/llm.ts

@@ -2,11 +2,13 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { Span } from '@opentelemetry/api';
 import { EventEmitter } from 'node:events';
 import { APIConnectionError, APIError } from '../_exceptions.js';
 import { log } from '../log.js';
 import type { LLMMetrics } from '../metrics/base.js';
-import type { APIConnectOptions } from '../types.js';
+import { recordException, traceTypes, tracer } from '../telemetry/index.js';
+import { type APIConnectOptions, intervalForRetry } from '../types.js';
 import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
 import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
 import type { ToolChoice, ToolContext } from './tool_context.js';
@@ -15,6 +17,7 @@ export interface ChoiceDelta {
   role: ChatRole;
   content?: string;
   toolCalls?: FunctionCall[];
+  extra?: Record<string, unknown>;
 }
 
 export interface CompletionUsage {
@@ -62,6 +65,18 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCal
     return 'unknown';
   }
 
+  /**
+   * Get the provider name for this LLM instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider(): string {
+    return 'unknown';
+  }
+
   /**
    * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
    */
@@ -78,7 +93,7 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCal
     connOptions?: APIConnectOptions;
     parallelToolCalls?: boolean;
     toolChoice?: ToolChoice;
-    extraKwargs?: Record<string, any>;
+    extraKwargs?: Record<string, unknown>;
   }): LLMStream;
 
   /**
@@ -104,6 +119,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   #llm: LLM;
   #chatCtx: ChatContext;
   #toolCtx?: ToolContext;
+  #llmRequestSpan?: Span;
 
   constructor(
     llm: LLM,
@@ -132,16 +148,30 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     // is run **after** the constructor has finished. Otherwise we get
     // runtime error when trying to access class variables in the
     // `run` method.
-    startSoon(() => this.mainTask().then(() => this.queue.close()));
+    startSoon(() => this.mainTask().finally(() => this.queue.close()));
   }
 
-  private async mainTask() {
+  private _mainTaskImpl = async (span: Span) => {
+    this.#llmRequestSpan = span;
+    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);
+
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
-        return await this.run();
+        return await tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              recordException(attemptSpan, toError(error));
+              throw error;
+            }
+          },
+          { name: 'llm_request_run' },
+        );
       } catch (error) {
         if (error instanceof APIError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+          const retryInterval = intervalForRetry(this._connOptions, i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -156,7 +186,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
             this.emitError({ error, recoverable: true });
             this.logger.warn(
               { llm: this.#llm.label(), attempt: i + 1, error },
-              `failed to generate LLM completion, retrying in ${retryInterval}s`,
+              `failed to generate LLM completion, retrying in ${retryInterval}ms`,
             );
           }
 
@@ -169,7 +199,13 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
         }
       }
     }
-  }
+  };
+
+  private mainTask = async () =>
+    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+      name: 'llm_request',
+      endOnExit: false,
+    });
 
   private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
     this.#llm.emit('error', {
@@ -186,6 +222,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     let ttft: bigint = BigInt(-1);
     let requestId = '';
     let usage: CompletionUsage | undefined;
+    let completionStartTime: string | undefined;
 
     for await (const ev of this.queue) {
       if (this.abortController.signal.aborted) {
@@ -195,6 +232,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
       requestId = ev.id;
       if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
+        completionStartTime = new Date().toISOString();
       }
       if (ev.usage) {
         usage = ev.usage;
@@ -203,12 +241,13 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     this.output.close();
 
     const duration = process.hrtime.bigint() - startTime;
+    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));
     const metrics: LLMMetrics = {
       type: 'llm_metrics',
       timestamp: Date.now(),
       requestId,
-      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),
-      duration: Math.trunc(Number(duration / BigInt(1000000))),
+      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),
+      durationMs,
       cancelled: this.abortController.signal.aborted,
       label: this.#llm.label(),
       completionTokens: usage?.completionTokens || 0,
@@ -216,10 +255,36 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
       promptCachedTokens: usage?.promptCachedTokens || 0,
       totalTokens: usage?.totalTokens || 0,
       tokensPerSecond: (() => {
-        const durationSeconds = Math.trunc(Number(duration / BigInt(1000000000)));
-        return durationSeconds > 0 ? (usage?.completionTokens || 0) / durationSeconds : 0;
+        if (durationMs <= 0) {
+          return 0;
+        }
+        return (usage?.completionTokens || 0) / (durationMs / 1000);
       })(),
+      metadata: {
+        modelProvider: this.#llm.provider,
+        modelName: this.#llm.model,
+      },
     };
+
+    if (this.#llmRequestSpan) {
+      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));
+
+      this.#llmRequestSpan.setAttributes({
+        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,
+        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,
+      });
+
+      if (completionStartTime) {
+        this.#llmRequestSpan.setAttribute(
+          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,
+          completionStartTime,
+        );
+      }
+
+      // End the span now that metrics are collected
+      this.#llmRequestSpan.end();
+    }
+
     this.#llm.emit('metrics_collected', metrics);
   }
 

package/src/llm/provider_format/google.test.ts

@@ -4,7 +4,12 @@
 import { VideoBufferType, VideoFrame } from '@livekit/rtc-node';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { initializeLogger } from '../../log.js';
-import { ChatContext, FunctionCall, FunctionCallOutput } from '../chat_context.js';
+import {
+  AgentHandoffItem,
+  ChatContext,
+  FunctionCall,
+  FunctionCallOutput,
+} from '../chat_context.js';
 import { serializeImage } from '../utils.js';
 import { toChatCtx } from './google.js';
 
@@ -769,4 +774,70 @@ describe('Google Provider Format - toChatCtx', () => {
     ]);
     expect(formatData.systemMessages).toBeNull();
   });
+
+  it('should filter out agent handoff items', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Hello' });
+
+    // Insert an agent handoff item
+    const handoff = new AgentHandoffItem({
+      oldAgentId: 'agent_1',
+      newAgentId: 'agent_2',
+    });
+    ctx.insert(handoff);
+
+    ctx.addMessage({ role: 'assistant', content: 'Hi there!' });
+
+    const [result, formatData] = await toChatCtx(ctx, false);
+
+    // Agent handoff should be filtered out, only messages should remain
+    expect(result).toEqual([
+      {
+        role: 'user',
+        parts: [{ text: 'Hello' }],
+      },
+      {
+        role: 'model',
+        parts: [{ text: 'Hi there!' }],
+      },
+    ]);
+    expect(formatData.systemMessages).toBeNull();
+  });
+
+  it('should handle multiple agent handoffs without errors', async () => {
+    const ctx = ChatContext.empty();
+
+    ctx.addMessage({ role: 'user', content: 'Start' });
+
+    // Multiple handoffs
+    ctx.insert(new AgentHandoffItem({ oldAgentId: undefined, newAgentId: 'agent_1' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 1' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 2' });
+
+    ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_2', newAgentId: 'agent_3' }));
+    ctx.addMessage({ role: 'assistant', content: 'Response from agent 3' });
+
+    const [result, formatData] = await toChatCtx(ctx, false);
+
+    // All handoffs should be filtered out
+    // Note: Google provider groups consecutive messages by the same role
+    expect(result).toEqual([
+      {
+        role: 'user',
+        parts: [{ text: 'Start' }],
+      },
+      {
+        role: 'model',
+        parts: [
+          { text: 'Response from agent 1' },
+          { text: 'Response from agent 2' },
+          { text: 'Response from agent 3' },
+        ],
+      },
+    ]);
+    expect(formatData.systemMessages).toBeNull();
+  });
 });

package/src/llm/provider_format/google.ts

@@ -12,11 +12,11 @@ export interface GoogleFormatData {
 export async function toChatCtx(
   chatCtx: ChatContext,
   injectDummyUserMessage: boolean = true,
-): Promise<[Record<string, any>[], GoogleFormatData]> {
-  const turns: Record<string, any>[] = [];
+): Promise<[Record<string, unknown>[], GoogleFormatData]> {
+  const turns: Record<string, unknown>[] = [];
   const systemMessages: string[] = [];
   let currentRole: string | null = null;
-  let parts: Record<string, any>[] = [];
+  let parts: Record<string, unknown>[] = [];
 
   // Flatten all grouped tool calls to get individual messages
   const itemGroups = groupToolCalls(chatCtx);
@@ -67,13 +67,17 @@ export async function toChatCtx(
         }
       }
     } else if (msg.type === 'function_call') {
-      parts.push({
+      const functionCallPart: Record<string, unknown> = {
         functionCall: {
           id: msg.callId,
           name: msg.name,
           args: JSON.parse(msg.args || '{}'),
         },
-      });
+      };
+      if (msg.thoughtSignature) {
+        functionCallPart.thoughtSignature = msg.thoughtSignature;
+      }
+      parts.push(functionCallPart);
     } else if (msg.type === 'function_call_output') {
       const response = msg.isError ? { error: msg.output } : { output: msg.output };
       parts.push({
@@ -104,7 +108,7 @@ export async function toChatCtx(
   ];
 }
 
-async function toImagePart(image: ImageContent): Promise<Record<string, any>> {
+async function toImagePart(image: ImageContent): Promise<Record<string, unknown>> {
   const cacheKey = 'serialized_image';
   if (!image._cache[cacheKey]) {
     image._cache[cacheKey] = await serializeImage(image);

package/src/llm/provider_format/index.ts

@@ -3,9 +3,12 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { ChatContext } from '../chat_context.js';
 import { toChatCtx as toChatCtxGoogle } from './google.js';
-import { toChatCtx as toChatCtxOpenai } from './openai.js';
+import {
+  toChatCtx as toChatCtxOpenai,
+  toResponsesChatCtx as toResponsesChatCtxOpenai,
+} from './openai.js';
 
-export type ProviderFormat = 'openai' | 'google';
+export type ProviderFormat = 'openai' | 'openai.responses' | 'google';
 
 export async function toChatCtx(
   format: ProviderFormat,
@@ -15,6 +18,8 @@ export async function toChatCtx(
   switch (format) {
     case 'openai':
       return await toChatCtxOpenai(chatCtx, injectDummyUserMessage);
+    case 'openai.responses':
+      return await toResponsesChatCtxOpenai(chatCtx, injectDummyUserMessage);
     case 'google':
       return await toChatCtxGoogle(chatCtx, injectDummyUserMessage);
     default: