@livekit/agents 0.7.6 → 0.7.7

package/dist/ipc/message.d.cts

@@ -0,0 +1,58 @@
+import type { RunningJobInfo } from '../job.js';
+import type { LoggerOptions } from '../log.js';
+export type IPCMessage = {
+    case: 'initializeRequest';
+    value: {
+        loggerOptions: LoggerOptions;
+        pingInterval?: number;
+        pingTimeout?: number;
+        highPingThreshold?: number;
+    };
+} | {
+    case: 'initializeResponse';
+    value: undefined;
+} | {
+    case: 'pingRequest';
+    value: {
+        timestamp: number;
+    };
+} | {
+    case: 'pongResponse';
+    value: {
+        lastTimestamp: number;
+        timestamp: number;
+    };
+} | {
+    case: 'startJobRequest';
+    value: {
+        runningJob: RunningJobInfo;
+    };
+} | {
+    case: 'shutdownRequest';
+    value: {
+        reason?: string;
+    };
+} | {
+    case: 'inferenceRequest';
+    value: {
+        method: string;
+        requestId: string;
+        data: unknown;
+    };
+} | {
+    case: 'inferenceResponse';
+    value: {
+        requestId: string;
+        data: unknown;
+        error?: Error;
+    };
+} | {
+    case: 'exiting';
+    value: {
+        reason?: string;
+    };
+} | {
+    case: 'done';
+    value: undefined;
+};
+//# sourceMappingURL=message.d.ts.map

package/dist/ipc/proc_pool.d.cts

@@ -0,0 +1,31 @@
+import { MultiMutex, Mutex } from '@livekit/mutex';
+import type { RunningJobInfo } from '../job.js';
+import { Queue } from '../utils.js';
+import type { InferenceExecutor } from './inference_executor.js';
+import type { JobExecutor } from './job_executor.js';
+export declare class ProcPool {
+    agent: string;
+    initializeTimeout: number;
+    closeTimeout: number;
+    executors: JobExecutor[];
+    tasks: Promise<void>[];
+    started: boolean;
+    closed: boolean;
+    controller: AbortController;
+    initMutex: Mutex;
+    procMutex?: MultiMutex;
+    procUnlock?: () => void;
+    warmedProcQueue: Queue<JobExecutor>;
+    inferenceExecutor?: InferenceExecutor;
+    memoryWarnMB: number;
+    memoryLimitMB: number;
+    constructor(agent: string, numIdleProcesses: number, initializeTimeout: number, closeTimeout: number, inferenceExecutor: InferenceExecutor | undefined, memoryWarnMB: number, memoryLimitMB: number);
+    get processes(): JobExecutor[];
+    getByJobId(id: string): JobExecutor | null;
+    launchJob(info: RunningJobInfo): Promise<void>;
+    procWatchTask(): Promise<void>;
+    start(): void;
+    run(signal: AbortSignal): Promise<void>;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=proc_pool.d.ts.map

package/dist/ipc/supervised_proc.d.cts

@@ -0,0 +1,30 @@
+/// <reference types="node" resolution-mode="require"/>
+import type { ChildProcess } from 'node:child_process';
+import type { RunningJobInfo } from '../job.js';
+import { Future } from '../utils.js';
+export interface ProcOpts {
+    initializeTimeout: number;
+    closeTimeout: number;
+    memoryWarnMB: number;
+    memoryLimitMB: number;
+    pingInterval: number;
+    pingTimeout: number;
+    highPingThreshold: number;
+}
+export declare abstract class SupervisedProc {
+    #private;
+    proc?: ChildProcess;
+    protected init: Future;
+    constructor(initializeTimeout: number, closeTimeout: number, memoryWarnMB: number, memoryLimitMB: number, pingInterval: number, pingTimeout: number, highPingThreshold: number);
+    abstract createProcess(): ChildProcess;
+    abstract mainTask(child: ChildProcess): Promise<void>;
+    get started(): boolean;
+    get runningJob(): RunningJobInfo | undefined;
+    start(): Promise<void>;
+    run(): Promise<void>;
+    join(): Promise<void>;
+    initialize(): Promise<void>;
+    close(): Promise<void>;
+    launchJob(info: RunningJobInfo): Promise<void>;
+}
+//# sourceMappingURL=supervised_proc.d.ts.map

package/dist/job.d.cts

@@ -0,0 +1,113 @@
+import type * as proto from '@livekit/protocol';
+import type { E2EEOptions, LocalParticipant, RemoteParticipant, Room, RtcConfiguration } from '@livekit/rtc-node';
+import type { InferenceExecutor } from './ipc/inference_executor.js';
+export declare class CurrentJobContext {
+    #private;
+    constructor(proc: JobContext);
+    static getCurrent(): JobContext;
+}
+/** Which tracks, if any, should the agent automatically subscribe to? */
+export declare enum AutoSubscribe {
+    SUBSCRIBE_ALL = 0,
+    SUBSCRIBE_NONE = 1,
+    VIDEO_ONLY = 2,
+    AUDIO_ONLY = 3
+}
+export type JobAcceptArguments = {
+    name: string;
+    identity: string;
+    metadata: string;
+    attributes?: {
+        [key: string]: string;
+    };
+};
+export type RunningJobInfo = {
+    acceptArguments: JobAcceptArguments;
+    job: proto.Job;
+    url: string;
+    token: string;
+};
+/** Attempted to add a function callback, but the function already exists. */
+export declare class FunctionExistsError extends Error {
+    constructor(msg?: string);
+}
+/** The job and environment context as seen by the agent, accessible by the entrypoint function. */
+export declare class JobContext {
+    #private;
+    /** @internal */
+    shutdownCallbacks: (() => Promise<void>)[];
+    constructor(proc: JobProcess, info: RunningJobInfo, room: Room, onConnect: () => void, onShutdown: (s: string) => void, inferenceExecutor: InferenceExecutor);
+    get proc(): JobProcess;
+    get job(): proto.Job;
+    /** @returns The room the agent was called into */
+    get room(): Room;
+    /** @returns The agent's participant if connected to the room, otherwise `undefined` */
+    get agent(): LocalParticipant | undefined;
+    /** @returns The global inference executor */
+    get inferenceExecutor(): InferenceExecutor;
+    /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */
+    addShutdownCallback(callback: () => Promise<void>): void;
+    waitForParticipant(identity?: string): Promise<RemoteParticipant>;
+    /**
+     * Connects the agent to the room.
+     *
+     * @remarks
+     * It is recommended to run this command as early in the function as possible, as executing it
+     * later may cause noticeable delay between user and agent joins.
+     *
+     * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |
+     * @livekit/rtc-node} for more information about the parameters.
+     */
+    connect(e2ee?: E2EEOptions, autoSubscribe?: AutoSubscribe, rtcConfig?: RtcConfiguration): Promise<void>;
+    /**
+     * Gracefully shuts down the job, and runs all shutdown promises.
+     *
+     * @param reason - Optional reason for shutdown
+     */
+    shutdown(reason?: string): void;
+    /** @internal */
+    onParticipantConnected(p: RemoteParticipant): void;
+    /**
+     * Adds a promise to be awaited whenever a new participant joins the room.
+     *
+     * @throws {@link FunctionExistsError} if an entrypoint already exists
+     */
+    addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>): void;
+}
+export declare class JobProcess {
+    #private;
+    userData: {
+        [id: string]: unknown;
+    };
+    get pid(): number;
+}
+/**
+ * A request sent by the server to spawn a new agent job.
+ *
+ * @remarks
+ * For most applications, this is best left to the default, which simply accepts the job and
+ * handles the logic inside the entrypoint function. This class is useful for vetting which
+ * requests should fill idle processes and which should be outright rejected.
+ */
+export declare class JobRequest {
+    #private;
+    /** @internal */
+    constructor(job: proto.Job, onReject: () => Promise<void>, onAccept: (args: JobAcceptArguments) => Promise<void>);
+    /** @returns The ID of the job, set by the LiveKit server */
+    get id(): string;
+    /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+    get job(): proto.Job;
+    /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+    get room(): proto.Room | undefined;
+    /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+    get publisher(): proto.ParticipantInfo | undefined;
+    /** @returns The agent's name, as set in {@link WorkerOptions} */
+    get agentName(): string;
+    /** Rejects the job. */
+    reject(): Promise<void>;
+    /** Accepts the job, launching it on an idle child process. */
+    accept(name?: string, identity?: string, metadata?: string, attributes?: {
+        [key: string]: string;
+    }): Promise<void>;
+}
+//# sourceMappingURL=job.d.ts.map

package/dist/llm/chat_context.d.cts

@@ -0,0 +1,66 @@
+import type { AudioFrame, VideoFrame } from '@livekit/rtc-node';
+import type { CallableFunctionResult, FunctionCallInfo } from './function_context.js';
+export declare enum ChatRole {
+    SYSTEM = 0,
+    USER = 1,
+    ASSISTANT = 2,
+    TOOL = 3
+}
+export interface ChatImage {
+    image: string | VideoFrame;
+    inferenceWidth?: number;
+    inferenceHeight?: number;
+    /**
+     * @internal
+     * Used by LLM implementations to store a processed version of the image for later use.
+     */
+    cache: {
+        [id: string | number | symbol]: any;
+    };
+}
+export interface ChatAudio {
+    frame: AudioFrame | AudioFrame[];
+}
+export type ChatContent = string | ChatImage | ChatAudio;
+export declare class ChatMessage {
+    readonly role: ChatRole;
+    readonly id?: string;
+    readonly name?: string;
+    readonly content?: ChatContent | ChatContent[];
+    readonly toolCalls?: FunctionCallInfo[];
+    readonly toolCallId?: string;
+    readonly toolException?: Error;
+    /** @internal */
+    constructor({ role, id, name, content, toolCalls, toolCallId, toolException, }: {
+        role: ChatRole;
+        id?: string;
+        name?: string;
+        content?: ChatContent | ChatContent[];
+        toolCalls?: FunctionCallInfo[];
+        toolCallId?: string;
+        toolException?: Error;
+    });
+    static createToolFromFunctionResult(func: CallableFunctionResult): ChatMessage;
+    static createToolCalls(toolCalls: FunctionCallInfo[], text?: string): ChatMessage;
+    static create(options: Partial<{
+        text?: string;
+        images: ChatImage[];
+        role: ChatRole;
+    }>): ChatMessage;
+    /** Returns a structured clone of this message. */
+    copy(): ChatMessage;
+}
+export declare class ChatContext {
+    messages: ChatMessage[];
+    metadata: {
+        [id: string]: any;
+    };
+    append(msg: {
+        text?: string;
+        images?: ChatImage[];
+        role: ChatRole;
+    }): ChatContext;
+    /** Returns a structured clone of this context. */
+    copy(): ChatContext;
+}
+//# sourceMappingURL=chat_context.d.ts.map

package/dist/llm/function_context.d.cts

@@ -0,0 +1,47 @@
+import { z } from 'zod';
+/** Type reinforcement for the callable function's execute parameters. */
+export type inferParameters<P extends z.ZodTypeAny> = z.infer<P>;
+/** Raw OpenAI-adherent function parameters. */
+export type OpenAIFunctionParameters = {
+    type: 'object';
+    properties: {
+        [id: string]: any;
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+/** A definition for a function callable by the LLM. */
+export interface CallableFunction<P extends z.ZodTypeAny = any, R = any> {
+    description: string;
+    parameters: OpenAIFunctionParameters | P;
+    execute: (args: inferParameters<P>) => PromiseLike<R>;
+}
+/** A function that has been called but is not yet running */
+export interface FunctionCallInfo<P extends z.ZodTypeAny = any, R = any> {
+    name: string;
+    func: CallableFunction<P, R>;
+    toolCallId: string;
+    rawParams: string;
+    params: inferParameters<P>;
+    task?: PromiseLike<CallableFunctionResult>;
+}
+/** The result of a ran FunctionCallInfo. */
+export interface CallableFunctionResult {
+    name: string;
+    toolCallId: string;
+    result?: any;
+    error?: any;
+}
+/** An object containing callable functions and their names */
+export type FunctionContext = {
+    [name: string]: CallableFunction;
+};
+/** @internal */
+export declare const oaiParams: (p: z.AnyZodObject) => {
+    type: "object";
+    properties: Record<string, any>;
+    required: string[];
+};
+/** @internal */
+export declare const oaiBuildFunctionInfo: (fncCtx: FunctionContext, toolCallId: string, fncName: string, rawArgs: string) => FunctionCallInfo;
+//# sourceMappingURL=function_context.d.ts.map

package/dist/llm/index.d.cts

@@ -0,0 +1,4 @@
+export { type CallableFunction, type FunctionCallInfo, type CallableFunctionResult, type FunctionContext, type inferParameters, oaiParams, oaiBuildFunctionInfo, } from './function_context.js';
+export { type ChatImage, type ChatAudio, type ChatContent, ChatRole, ChatMessage, ChatContext, } from './chat_context.js';
+export { type ChoiceDelta, type CompletionUsage, type Choice, type ChatChunk, type LLMCallbacks, LLMEvent, LLM, LLMStream, } from './llm.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/llm/llm.cjs

@@ -47,13 +47,13 @@ class LLMStream {
   }
   async monitorMetrics() {
     const startTime = process.hrtime.bigint();
-    let ttft;
+    let ttft = BigInt(-1);
     let requestId = "";
     let usage;
     for await (const ev of this.queue) {
       this.output.put(ev);
       requestId = ev.requestId;
-      if (!ttft) {
+      if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
       }
       if (ev.usage) {
@@ -65,7 +65,7 @@ class LLMStream {
     const metrics = {
       timestamp: Date.now(),
       requestId,
-      ttft: Math.trunc(Number(ttft / BigInt(1e6))),
+      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1e6))),
       duration: Math.trunc(Number(duration / BigInt(1e6))),
       cancelled: false,
       // XXX(nbsp)

package/dist/llm/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { ChatContext, ChatRole } from './chat_context.js';\nimport type { FunctionCallInfo, FunctionContext } from './function_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCallInfo[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n}\n\nexport interface Choice {\n  delta: ChoiceDelta;\n  index: number;\n}\n\nexport interface ChatChunk {\n  requestId: string;\n  choices: Choice[];\n  usage?: CompletionUsage;\n}\n\nexport enum LLMEvent {\n  METRICS_COLLECTED,\n}\n\nexport type LLMCallbacks = {\n  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: ChatContext;\n    fncCtx?: FunctionContext;\n    temperature?: number;\n    n?: number;\n    parallelToolCalls?: boolean;\n  }): LLMStream;\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected _functionCalls: FunctionCallInfo[] = [];\n  abstract label: string;\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #fncCtx?: FunctionContext;\n\n  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint | undefined;\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      this.output.put(ev);\n      requestId = ev.requestId;\n      if (!ttft) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttft: Math.trunc(Number(ttft! / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** List of called functions from this stream. */\n  get functionCalls(): FunctionCallInfo[] {\n    return this._functionCalls;\n  }\n\n  /** The function context of this stream. */\n  get fncCtx(): FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** Execute all deferred functions of this stream concurrently. */\n  executeFunctions(): FunctionCallInfo[] {\n    this._functionCalls.forEach(\n      (f) =>\n        (f.task = f.func.execute(f.params).then(\n          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),\n          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),\n        )),\n    );\n    return this._functionCalls;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.output.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAA6B;AAE7B,mBAAmC;AA2B5B,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAQL,MAAe,YAAa,gCAAsD;AAiBzF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,iBAAqC,CAAC;AAAA,EAGhD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,SAAsB,QAA0B;AACpE,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI;AACJ,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,KAAK,MAAM,OAAO,OAAQ,OAAO,GAAO,CAAC,CAAC;AAAA,MAChD,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,IAAI,gBAAoC;AACtC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,mBAAuC;AACrC,SAAK,eAAe;AAAA,MAClB,CAAC,MACE,EAAE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE;AAAA,QACjC,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,OAAO;AAAA,QAC9D,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,MAAM;AAAA,MAC9D;AAAA,IACJ;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,MAAM;AAClB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["LLMEvent"]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { ChatContext, ChatRole } from './chat_context.js';\nimport type { FunctionCallInfo, FunctionContext } from './function_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCallInfo[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n}\n\nexport interface Choice {\n  delta: ChoiceDelta;\n  index: number;\n}\n\nexport interface ChatChunk {\n  requestId: string;\n  choices: Choice[];\n  usage?: CompletionUsage;\n}\n\nexport enum LLMEvent {\n  METRICS_COLLECTED,\n}\n\nexport type LLMCallbacks = {\n  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: ChatContext;\n    fncCtx?: FunctionContext;\n    temperature?: number;\n    n?: number;\n    parallelToolCalls?: boolean;\n  }): LLMStream;\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected _functionCalls: FunctionCallInfo[] = [];\n  abstract label: string;\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #fncCtx?: FunctionContext;\n\n  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      this.output.put(ev);\n      requestId = ev.requestId;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** List of called functions from this stream. */\n  get functionCalls(): FunctionCallInfo[] {\n    return this._functionCalls;\n  }\n\n  /** The function context of this stream. */\n  get fncCtx(): FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** Execute all deferred functions of this stream concurrently. */\n  executeFunctions(): FunctionCallInfo[] {\n    this._functionCalls.forEach(\n      (f) =>\n        (f.task = f.func.execute(f.params).then(\n          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),\n          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),\n        )),\n    );\n    return this._functionCalls;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.output.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAA6B;AAE7B,mBAAmC;AA2B5B,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAQL,MAAe,YAAa,gCAAsD;AAiBzF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,gCAA8B;AAAA,EAC3C,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,iBAAqC,CAAC;AAAA,EAGhD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,SAAsB,QAA0B;AACpE,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,IAAI,gBAAoC;AACtC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,mBAAuC;AACrC,SAAK,eAAe;AAAA,MAClB,CAAC,MACE,EAAE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE;AAAA,QACjC,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,OAAO;AAAA,QAC9D,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,MAAM;AAAA,MAC9D;AAAA,IACJ;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,MAAM;AAClB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["LLMEvent"]}

package/dist/llm/llm.d.cts

@@ -0,0 +1,66 @@
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { LLMMetrics } from '../metrics/base.js';
+import { AsyncIterableQueue } from '../utils.js';
+import type { ChatContext, ChatRole } from './chat_context.js';
+import type { FunctionCallInfo, FunctionContext } from './function_context.js';
+export interface ChoiceDelta {
+    role: ChatRole;
+    content?: string;
+    toolCalls?: FunctionCallInfo[];
+}
+export interface CompletionUsage {
+    completionTokens: number;
+    promptTokens: number;
+    totalTokens: number;
+}
+export interface Choice {
+    delta: ChoiceDelta;
+    index: number;
+}
+export interface ChatChunk {
+    requestId: string;
+    choices: Choice[];
+    usage?: CompletionUsage;
+}
+export declare enum LLMEvent {
+    METRICS_COLLECTED = 0
+}
+export type LLMCallbacks = {
+    [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;
+};
+declare const LLM_base: new () => TypedEmitter<LLMCallbacks>;
+export declare abstract class LLM extends LLM_base {
+    /**
+     * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
+     */
+    abstract chat({ chatCtx, fncCtx, temperature, n, parallelToolCalls, }: {
+        chatCtx: ChatContext;
+        fncCtx?: FunctionContext;
+        temperature?: number;
+        n?: number;
+        parallelToolCalls?: boolean;
+    }): LLMStream;
+}
+export declare abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+    #private;
+    protected output: AsyncIterableQueue<ChatChunk>;
+    protected queue: AsyncIterableQueue<ChatChunk>;
+    protected closed: boolean;
+    protected _functionCalls: FunctionCallInfo[];
+    abstract label: string;
+    constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext);
+    protected monitorMetrics(): Promise<void>;
+    /** List of called functions from this stream. */
+    get functionCalls(): FunctionCallInfo[];
+    /** The function context of this stream. */
+    get fncCtx(): FunctionContext | undefined;
+    /** The initial chat context of this stream. */
+    get chatCtx(): ChatContext;
+    /** Execute all deferred functions of this stream concurrently. */
+    executeFunctions(): FunctionCallInfo[];
+    next(): Promise<IteratorResult<ChatChunk>>;
+    close(): void;
+    [Symbol.asyncIterator](): LLMStream;
+}
+export {};
+//# sourceMappingURL=llm.d.ts.map

package/dist/llm/llm.js

@@ -22,13 +22,13 @@ class LLMStream {
   }
   async monitorMetrics() {
     const startTime = process.hrtime.bigint();
-    let ttft;
+    let ttft = BigInt(-1);
     let requestId = "";
     let usage;
     for await (const ev of this.queue) {
       this.output.put(ev);
       requestId = ev.requestId;
-      if (!ttft) {
+      if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
       }
       if (ev.usage) {
@@ -40,7 +40,7 @@ class LLMStream {
     const metrics = {
       timestamp: Date.now(),
       requestId,
-      ttft: Math.trunc(Number(ttft / BigInt(1e6))),
+      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1e6))),
       duration: Math.trunc(Number(duration / BigInt(1e6))),
       cancelled: false,
       // XXX(nbsp)

package/dist/llm/llm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { ChatContext, ChatRole } from './chat_context.js';\nimport type { FunctionCallInfo, FunctionContext } from './function_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCallInfo[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n}\n\nexport interface Choice {\n  delta: ChoiceDelta;\n  index: number;\n}\n\nexport interface ChatChunk {\n  requestId: string;\n  choices: Choice[];\n  usage?: CompletionUsage;\n}\n\nexport enum LLMEvent {\n  METRICS_COLLECTED,\n}\n\nexport type LLMCallbacks = {\n  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: ChatContext;\n    fncCtx?: FunctionContext;\n    temperature?: number;\n    n?: number;\n    parallelToolCalls?: boolean;\n  }): LLMStream;\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected _functionCalls: FunctionCallInfo[] = [];\n  abstract label: string;\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #fncCtx?: FunctionContext;\n\n  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint | undefined;\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      this.output.put(ev);\n      requestId = ev.requestId;\n      if (!ttft) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttft: Math.trunc(Number(ttft! / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** List of called functions from this stream. */\n  get functionCalls(): FunctionCallInfo[] {\n    return this._functionCalls;\n  }\n\n  /** The function context of this stream. */\n  get fncCtx(): FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** Execute all deferred functions of this stream concurrently. */\n  executeFunctions(): FunctionCallInfo[] {\n    this._functionCalls.forEach(\n      (f) =>\n        (f.task = f.func.execute(f.params).then(\n          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),\n          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),\n        )),\n    );\n    return this._functionCalls;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.output.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,0BAA0B;AA2B5B,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAQL,MAAe,YAAa,aAAsD;AAiBzF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,iBAAqC,CAAC;AAAA,EAGhD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,SAAsB,QAA0B;AACpE,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI;AACJ,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,KAAK,MAAM,OAAO,OAAQ,OAAO,GAAO,CAAC,CAAC;AAAA,MAChD,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,IAAI,gBAAoC;AACtC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,mBAAuC;AACrC,SAAK,eAAe;AAAA,MAClB,CAAC,MACE,EAAE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE;AAAA,QACjC,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,OAAO;AAAA,QAC9D,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,MAAM;AAAA,MAC9D;AAAA,IACJ;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,MAAM;AAClB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["LLMEvent"]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { ChatContext, ChatRole } from './chat_context.js';\nimport type { FunctionCallInfo, FunctionContext } from './function_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCallInfo[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n}\n\nexport interface Choice {\n  delta: ChoiceDelta;\n  index: number;\n}\n\nexport interface ChatChunk {\n  requestId: string;\n  choices: Choice[];\n  usage?: CompletionUsage;\n}\n\nexport enum LLMEvent {\n  METRICS_COLLECTED,\n}\n\nexport type LLMCallbacks = {\n  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: ChatContext;\n    fncCtx?: FunctionContext;\n    temperature?: number;\n    n?: number;\n    parallelToolCalls?: boolean;\n  }): LLMStream;\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected _functionCalls: FunctionCallInfo[] = [];\n  abstract label: string;\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #fncCtx?: FunctionContext;\n\n  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      this.output.put(ev);\n      requestId = ev.requestId;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** List of called functions from this stream. */\n  get functionCalls(): FunctionCallInfo[] {\n    return this._functionCalls;\n  }\n\n  /** The function context of this stream. */\n  get fncCtx(): FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** Execute all deferred functions of this stream concurrently. */\n  executeFunctions(): FunctionCallInfo[] {\n    this._functionCalls.forEach(\n      (f) =>\n        (f.task = f.func.execute(f.params).then(\n          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),\n          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),\n        )),\n    );\n    return this._functionCalls;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.output.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,0BAA0B;AA2B5B,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAQL,MAAe,YAAa,aAAsD;AAiBzF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,iBAAqC,CAAC;AAAA,EAGhD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,SAAsB,QAA0B;AACpE,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,IAAI,gBAAoC;AACtC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,mBAAuC;AACrC,SAAK,eAAe;AAAA,MAClB,CAAC,MACE,EAAE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE;AAAA,QACjC,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,OAAO;AAAA,QAC9D,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,MAAM;AAAA,MAC9D;AAAA,IACJ;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,MAAM;AAClB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["LLMEvent"]}

package/dist/log.d.cts

@@ -0,0 +1,13 @@
+import type { Logger } from 'pino';
+/** @internal */
+export type LoggerOptions = {
+    pretty: boolean;
+    level?: string;
+};
+/** @internal */
+export declare let loggerOptions: LoggerOptions;
+/** @internal */
+export declare const log: () => Logger;
+/** @internal */
+export declare const initializeLogger: ({ pretty, level }: LoggerOptions) => void;
+//# sourceMappingURL=log.d.ts.map

package/dist/metrics/base.d.cts

@@ -0,0 +1,96 @@
+export interface LLMMetrics {
+    requestId: string;
+    timestamp: number;
+    ttft: number;
+    duration: number;
+    label: string;
+    cancelled: boolean;
+    completionTokens: number;
+    promptTokens: number;
+    totalTokens: number;
+    tokensPerSecond: number;
+    error?: Error;
+}
+export interface STTMetrics {
+    requestId: string;
+    timestamp: number;
+    duration: number;
+    label: string;
+    audioDuration: number;
+    streamed: boolean;
+    error?: Error;
+}
+export interface TTSMetrics {
+    requestId: string;
+    timestamp: number;
+    ttfb: number;
+    duration: number;
+    label: string;
+    audioDuration: number;
+    cancelled: boolean;
+    charactersCount: number;
+    streamed: boolean;
+    error?: Error;
+}
+export interface VADMetrics {
+    timestamp: number;
+    idleTime: number;
+    inferenceDurationTotal: number;
+    inferenceCount: number;
+    label: string;
+}
+export interface PipelineEOUMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+    /** Timestamp of when the event was recorded */
+    timestamp: number;
+    /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
+    endOfUtteranceDelay: number;
+    /**
+     * Time taken to obtain the transcript after the end of the user's speech.
+     *
+     * @remarks
+     * May be 0 if the transcript was already available.
+     */
+    transcriptionDelay: number;
+}
+export interface PipelineLLMMetrics extends LLMMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+}
+export interface PipelineTTSMetrics extends TTSMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+}
+export type PipelineSTTMetrics = STTMetrics;
+export type PipelineVADMetrics = VADMetrics;
+export declare class MultimodalLLMError extends Error {
+    type?: string;
+    reason?: string;
+    code?: string;
+    constructor({ type, reason, code, message, }?: {
+        type?: string;
+        reason?: string;
+        code?: string;
+        message?: string;
+    });
+}
+export interface MultimodalLLMMetrics extends LLMMetrics {
+    inputTokenDetails: {
+        cachedTokens: number;
+        textTokens: number;
+        audioTokens: number;
+    };
+    outputTokenDetails: {
+        textTokens: number;
+        audioTokens: number;
+    };
+}
+export type AgentMetrics = STTMetrics | LLMMetrics | TTSMetrics | VADMetrics | PipelineSTTMetrics | PipelineEOUMetrics | PipelineLLMMetrics | PipelineTTSMetrics | PipelineVADMetrics | MultimodalLLMMetrics;
+//# sourceMappingURL=base.d.ts.map

package/dist/metrics/index.d.cts

@@ -0,0 +1,5 @@
+export type { AgentMetrics, STTMetrics, LLMMetrics, TTSMetrics, VADMetrics, PipelineSTTMetrics, PipelineEOUMetrics, PipelineLLMMetrics, PipelineTTSMetrics, PipelineVADMetrics, MultimodalLLMMetrics, } from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/usage_collector.d.cts

@@ -0,0 +1,14 @@
+import type { AgentMetrics } from './base.js';
+export interface UsageSummary {
+    llmPromptTokens: number;
+    llmCompletionTokens: number;
+    ttsCharactersCount: number;
+    sttAudioDuration: number;
+}
+export declare class UsageCollector {
+    #private;
+    constructor();
+    collect(metrics: AgentMetrics): void;
+    get summary(): UsageSummary;
+}
+//# sourceMappingURL=usage_collector.d.ts.map