@livekit/agents 0.7.9 → 1.0.0-next.1

package/src/llm/index.ts

@@ -2,31 +2,66 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 export {
-  type CallableFunction,
-  type FunctionCallInfo,
-  type CallableFunctionResult,
-  type FunctionContext,
-  type inferParameters,
-  oaiParams,
-  oaiBuildFunctionInfo,
-} from './function_context.js';
+  handoff,
+  isFunctionTool,
+  tool,
+  ToolError,
+  type AgentHandoff,
+  type FunctionTool,
+  type ProviderDefinedTool,
+  type Tool,
+  type ToolChoice,
+  type ToolContext,
+  type ToolOptions,
+  type ToolType,
+} from './tool_context.js';
 
 export {
-  type ChatImage,
-  type ChatAudio,
-  type ChatContent,
-  ChatRole,
-  ChatMessage,
   ChatContext,
+  ChatMessage,
+  createAudioContent,
+  createImageContent,
+  FunctionCall,
+  FunctionCallOutput,
+  type AudioContent,
+  type ChatContent,
+  type ChatItem,
+  type ChatRole,
+  type ImageContent,
 } from './chat_context.js';
 
+export type { ProviderFormat } from './provider_format/index.js';
+
 export {
+  LLM,
+  LLMStream,
+  type ChatChunk,
   type ChoiceDelta,
   type CompletionUsage,
-  type Choice,
-  type ChatChunk,
   type LLMCallbacks,
-  LLMEvent,
-  LLM,
-  LLMStream,
 } from './llm.js';
+
+export {
+  RealtimeModel,
+  RealtimeSession,
+  type GenerationCreatedEvent,
+  type InputSpeechStartedEvent,
+  type InputSpeechStoppedEvent,
+  type InputTranscriptionCompleted,
+  type MessageGeneration,
+  type RealtimeCapabilities,
+  type RealtimeModelError,
+  type RealtimeSessionReconnectedEvent,
+} from './realtime.js';
+
+export { RemoteChatContext } from './remote_chat_context.js';
+
+export {
+  computeChatCtxDiff,
+  createToolOptions,
+  executeToolCall,
+  oaiBuildFunctionInfo,
+  oaiParams,
+  toJsonSchema,
+  type OpenAIFunctionParameters,
+} from './utils.js';

package/src/llm/llm.ts

@@ -1,79 +1,184 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import { EventEmitter } from 'node:events';
+import { APIConnectionError, APIError } from '../_exceptions.js';
+import { log } from '../log.js';
 import type { LLMMetrics } from '../metrics/base.js';
-import { AsyncIterableQueue } from '../utils.js';
-import type { ChatContext, ChatRole } from './chat_context.js';
-import type { FunctionCallInfo, FunctionContext } from './function_context.js';
+import type { APIConnectOptions } from '../types.js';
+import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
+import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
 
 export interface ChoiceDelta {
   role: ChatRole;
   content?: string;
-  toolCalls?: FunctionCallInfo[];
+  toolCalls?: FunctionCall[];
 }
 
 export interface CompletionUsage {
   completionTokens: number;
   promptTokens: number;
+  promptCachedTokens: number;
   totalTokens: number;
 }
 
-export interface Choice {
-  delta: ChoiceDelta;
-  index: number;
-}
-
 export interface ChatChunk {
-  requestId: string;
-  choices: Choice[];
+  id: string;
+  delta?: ChoiceDelta;
   usage?: CompletionUsage;
 }
 
-export enum LLMEvent {
-  METRICS_COLLECTED,
+export interface LLMError {
+  type: 'llm_error';
+  timestamp: number;
+  label: string;
+  error: Error;
+  recoverable: boolean;
 }
 
 export type LLMCallbacks = {
-  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+  ['metrics_collected']: (metrics: LLMMetrics) => void;
+  ['error']: (error: LLMError) => void;
 };
 
 export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {
+  constructor() {
+    super();
+  }
+
+  abstract label(): string;
+
+  /**
+   * Get the model name/identifier for this LLM instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model(): string {
+    return 'unknown';
+  }
+
   /**
    * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
    */
   abstract chat({
     chatCtx,
-    fncCtx,
-    temperature,
-    n,
+    toolCtx,
+    connOptions,
     parallelToolCalls,
+    toolChoice,
+    extraKwargs,
   }: {
     chatCtx: ChatContext;
-    fncCtx?: FunctionContext;
-    temperature?: number;
-    n?: number;
+    toolCtx?: ToolContext;
+    connOptions?: APIConnectOptions;
     parallelToolCalls?: boolean;
+    toolChoice?: ToolChoice;
+    extraKwargs?: Record<string, any>;
   }): LLMStream;
+
+  /**
+   * Pre-warm connection to the LLM service
+   */
+  prewarm(): void {
+    // Default implementation - subclasses can override
+  }
+
+  async aclose(): Promise<void> {
+    // Default implementation - subclasses can override
+  }
 }
 
 export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   protected output = new AsyncIterableQueue<ChatChunk>();
   protected queue = new AsyncIterableQueue<ChatChunk>();
   protected closed = false;
-  protected _functionCalls: FunctionCallInfo[] = [];
-  abstract label: string;
+  protected abortController = new AbortController();
+  protected _connOptions: APIConnectOptions;
+  protected logger = log();
 
   #llm: LLM;
   #chatCtx: ChatContext;
-  #fncCtx?: FunctionContext;
-
-  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {
+  #toolCtx?: ToolContext;
+
+  constructor(
+    llm: LLM,
+    {
+      chatCtx,
+      toolCtx,
+      connOptions,
+    }: {
+      chatCtx: ChatContext;
+      toolCtx?: ToolContext;
+      connOptions: APIConnectOptions;
+    },
+  ) {
     this.#llm = llm;
     this.#chatCtx = chatCtx;
-    this.#fncCtx = fncCtx;
+    this.#toolCtx = toolCtx;
+    this._connOptions = connOptions;
     this.monitorMetrics();
+    this.abortController.signal.addEventListener('abort', () => {
+      // TODO (AJS-37) clean this up when we refactor with streams
+      this.output.close();
+      this.closed = true;
+    });
+
+    // this is a hack to immitate asyncio.create_task so that mainTask
+    // is run **after** the constructor has finished. Otherwise we get
+    // runtime error when trying to access class variables in the
+    // `run` method.
+    startSoon(() => this.mainTask().then(() => this.queue.close()));
+  }
+
+  private async mainTask() {
+    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+      try {
+        return await this.run();
+      } catch (error) {
+        if (error instanceof APIError) {
+          const retryInterval = this._connOptions._intervalForRetry(i);
+
+          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+            this.emitError({ error, recoverable: false });
+            throw error;
+          } else if (i === this._connOptions.maxRetry) {
+            this.emitError({ error, recoverable: false });
+            throw new APIConnectionError({
+              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,
+              options: { retryable: false },
+            });
+          } else {
+            this.emitError({ error, recoverable: true });
+            this.logger.warn(
+              { llm: this.#llm.label(), attempt: i + 1, error },
+              `failed to generate LLM completion, retrying in ${retryInterval}s`,
+            );
+          }
+
+          if (retryInterval > 0) {
+            await delay(retryInterval);
+          }
+        } else {
+          this.emitError({ error: toError(error), recoverable: false });
+          throw error;
+        }
+      }
+    }
+  }
+
+  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
+    this.#llm.emit('error', {
+      type: 'llm_error',
+      timestamp: Date.now(),
+      label: this.#llm.label(),
+      error,
+      recoverable,
+    });
   }
 
   protected async monitorMetrics() {
@@ -83,8 +188,11 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     let usage: CompletionUsage | undefined;
 
     for await (const ev of this.queue) {
+      if (this.abortController.signal.aborted) {
+        break;
+      }
       this.output.put(ev);
-      requestId = ev.requestId;
+      requestId = ev.id;
       if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
       }
@@ -96,29 +204,28 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
 
     const duration = process.hrtime.bigint() - startTime;
     const metrics: LLMMetrics = {
+      type: 'llm_metrics',
       timestamp: Date.now(),
       requestId,
       ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),
       duration: Math.trunc(Number(duration / BigInt(1000000))),
-      cancelled: false, // XXX(nbsp)
-      label: this.label,
+      cancelled: this.abortController.signal.aborted,
+      label: this.#llm.label(),
       completionTokens: usage?.completionTokens || 0,
       promptTokens: usage?.promptTokens || 0,
+      promptCachedTokens: usage?.promptCachedTokens || 0,
       totalTokens: usage?.totalTokens || 0,
       tokensPerSecond:
         (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),
     };
-    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);
+    this.#llm.emit('metrics_collected', metrics);
   }
 
-  /** List of called functions from this stream. */
-  get functionCalls(): FunctionCallInfo[] {
-    return this._functionCalls;
-  }
+  protected abstract run(): Promise<void>;
 
   /** The function context of this stream. */
-  get fncCtx(): FunctionContext | undefined {
-    return this.#fncCtx;
+  get toolCtx(): ToolContext | undefined {
+    return this.#toolCtx;
   }
 
   /** The initial chat context of this stream. */
@@ -126,16 +233,9 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     return this.#chatCtx;
   }
 
-  /** Execute all deferred functions of this stream concurrently. */
-  executeFunctions(): FunctionCallInfo[] {
-    this._functionCalls.forEach(
-      (f) =>
-        (f.task = f.func.execute(f.params).then(
-          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),
-          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),
-        )),
-    );
-    return this._functionCalls;
+  /** The connection options for this stream. */
+  get connOptions(): APIConnectOptions {
+    return this._connOptions;
   }
 
   next(): Promise<IteratorResult<ChatChunk>> {
@@ -143,9 +243,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   }
 
   close() {
-    this.output.close();
-    this.queue.close();
-    this.closed = true;
+    this.abortController.abort();
   }
 
   [Symbol.asyncIterator](): LLMStream {