@livekit/agents 0.7.5 → 0.7.7

package/dist/llm/llm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { ChatContext, ChatRole } from './chat_context.js';\nimport type { FunctionCallInfo, FunctionContext } from './function_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCallInfo[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n}\n\nexport interface Choice {\n  delta: ChoiceDelta;\n  index: number;\n}\n\nexport interface ChatChunk {\n  requestId: string;\n  choices: Choice[];\n  usage?: CompletionUsage;\n}\n\nexport enum LLMEvent {\n  METRICS_COLLECTED,\n}\n\nexport type LLMCallbacks = {\n  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: ChatContext;\n    fncCtx?: FunctionContext;\n    temperature?: number;\n    n?: number;\n    parallelToolCalls?: boolean;\n  }): LLMStream;\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected _functionCalls: FunctionCallInfo[] = [];\n  abstract label: string;\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #fncCtx?: FunctionContext;\n\n  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint | undefined;\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      this.output.put(ev);\n      requestId = ev.requestId;\n      if (!ttft) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttft: Math.trunc(Number(ttft! / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** List of called functions from this stream. */\n  get functionCalls(): FunctionCallInfo[] {\n    return this._functionCalls;\n  }\n\n  /** The function context of this stream. */\n  get fncCtx(): FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** Execute all deferred functions of this stream concurrently. */\n  executeFunctions(): FunctionCallInfo[] {\n    this._functionCalls.forEach(\n      (f) =>\n        (f.task = f.func.execute(f.params).then(\n          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),\n          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),\n        )),\n    );\n    return this._functionCalls;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.output.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,0BAA0B;AA2B5B,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAQL,MAAe,YAAa,aAAsD;AAiBzF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,iBAAqC,CAAC;AAAA,EAGhD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,SAAsB,QAA0B;AACpE,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI;AACJ,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,CAAC,MAAM;AACT,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,KAAK,MAAM,OAAO,OAAQ,OAAO,GAAO,CAAC,CAAC;AAAA,MAChD,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,IAAI,gBAAoC;AACtC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,mBAAuC;AACrC,SAAK,eAAe;AAAA,MAClB,CAAC,MACE,EAAE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE;AAAA,QACjC,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,OAAO;AAAA,QAC9D,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,MAAM;AAAA,MAC9D;AAAA,IACJ;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,MAAM;AAClB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["LLMEvent"]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { ChatContext, ChatRole } from './chat_context.js';\nimport type { FunctionCallInfo, FunctionContext } from './function_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCallInfo[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  totalTokens: number;\n}\n\nexport interface Choice {\n  delta: ChoiceDelta;\n  index: number;\n}\n\nexport interface ChatChunk {\n  requestId: string;\n  choices: Choice[];\n  usage?: CompletionUsage;\n}\n\nexport enum LLMEvent {\n  METRICS_COLLECTED,\n}\n\nexport type LLMCallbacks = {\n  [LLMEvent.METRICS_COLLECTED]: (metrics: LLMMetrics) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    fncCtx,\n    temperature,\n    n,\n    parallelToolCalls,\n  }: {\n    chatCtx: ChatContext;\n    fncCtx?: FunctionContext;\n    temperature?: number;\n    n?: number;\n    parallelToolCalls?: boolean;\n  }): LLMStream;\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected _functionCalls: FunctionCallInfo[] = [];\n  abstract label: string;\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #fncCtx?: FunctionContext;\n\n  constructor(llm: LLM, chatCtx: ChatContext, fncCtx?: FunctionContext) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#fncCtx = fncCtx;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      this.output.put(ev);\n      requestId = ev.requestId;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const metrics: LLMMetrics = {\n      timestamp: Date.now(),\n      requestId,\n      ttft: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      duration: Math.trunc(Number(duration / BigInt(1000000))),\n      cancelled: false, // XXX(nbsp)\n      label: this.label,\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond:\n        (usage?.completionTokens || 0) / Math.trunc(Number(duration / BigInt(1000000000))),\n    };\n    this.#llm.emit(LLMEvent.METRICS_COLLECTED, metrics);\n  }\n\n  /** List of called functions from this stream. */\n  get functionCalls(): FunctionCallInfo[] {\n    return this._functionCalls;\n  }\n\n  /** The function context of this stream. */\n  get fncCtx(): FunctionContext | undefined {\n    return this.#fncCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** Execute all deferred functions of this stream concurrently. */\n  executeFunctions(): FunctionCallInfo[] {\n    this._functionCalls.forEach(\n      (f) =>\n        (f.task = f.func.execute(f.params).then(\n          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),\n          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),\n        )),\n    );\n    return this._functionCalls;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.output.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,0BAA0B;AA2B5B,IAAK,WAAL,kBAAKA,cAAL;AACL,EAAAA,oBAAA;AADU,SAAAA;AAAA,GAAA;AAQL,MAAe,YAAa,aAAsD;AAiBzF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,iBAAqC,CAAC;AAAA,EAGhD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,SAAsB,QAA0B;AACpE,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,UAAU;AACf,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,UAAsB;AAAA,MAC1B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,MAAM,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC1E,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,MACvD,WAAW;AAAA;AAAA,MACX,OAAO,KAAK;AAAA,MACZ,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,cAAa,+BAAO,gBAAe;AAAA,MACnC,mBACG,+BAAO,qBAAoB,KAAK,KAAK,MAAM,OAAO,WAAW,OAAO,GAAU,CAAC,CAAC;AAAA,IACrF;AACA,SAAK,KAAK,KAAK,2BAA4B,OAAO;AAAA,EACpD;AAAA;AAAA,EAGA,IAAI,gBAAoC;AACtC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,mBAAuC;AACrC,SAAK,eAAe;AAAA,MAClB,CAAC,MACE,EAAE,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE;AAAA,QACjC,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,OAAO;AAAA,QAC9D,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,EAAE,YAAY,MAAM;AAAA,MAC9D;AAAA,IACJ;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,MAAM;AAClB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["LLMEvent"]}

package/dist/log.d.cts

@@ -0,0 +1,13 @@
+import type { Logger } from 'pino';
+/** @internal */
+export type LoggerOptions = {
+    pretty: boolean;
+    level?: string;
+};
+/** @internal */
+export declare let loggerOptions: LoggerOptions;
+/** @internal */
+export declare const log: () => Logger;
+/** @internal */
+export declare const initializeLogger: ({ pretty, level }: LoggerOptions) => void;
+//# sourceMappingURL=log.d.ts.map

package/dist/metrics/base.d.cts

@@ -0,0 +1,96 @@
+export interface LLMMetrics {
+    requestId: string;
+    timestamp: number;
+    ttft: number;
+    duration: number;
+    label: string;
+    cancelled: boolean;
+    completionTokens: number;
+    promptTokens: number;
+    totalTokens: number;
+    tokensPerSecond: number;
+    error?: Error;
+}
+export interface STTMetrics {
+    requestId: string;
+    timestamp: number;
+    duration: number;
+    label: string;
+    audioDuration: number;
+    streamed: boolean;
+    error?: Error;
+}
+export interface TTSMetrics {
+    requestId: string;
+    timestamp: number;
+    ttfb: number;
+    duration: number;
+    label: string;
+    audioDuration: number;
+    cancelled: boolean;
+    charactersCount: number;
+    streamed: boolean;
+    error?: Error;
+}
+export interface VADMetrics {
+    timestamp: number;
+    idleTime: number;
+    inferenceDurationTotal: number;
+    inferenceCount: number;
+    label: string;
+}
+export interface PipelineEOUMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+    /** Timestamp of when the event was recorded */
+    timestamp: number;
+    /** Amount of time between the end of speech from VAD and the decision to end the user's turn */
+    endOfUtteranceDelay: number;
+    /**
+     * Time taken to obtain the transcript after the end of the user's speech.
+     *
+     * @remarks
+     * May be 0 if the transcript was already available.
+     */
+    transcriptionDelay: number;
+}
+export interface PipelineLLMMetrics extends LLMMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+}
+export interface PipelineTTSMetrics extends TTSMetrics {
+    /**
+     * Unique identifier shared across different metrics to combine related STT, LLM, and TTS metrics
+     */
+    sequenceId: string;
+}
+export type PipelineSTTMetrics = STTMetrics;
+export type PipelineVADMetrics = VADMetrics;
+export declare class MultimodalLLMError extends Error {
+    type?: string;
+    reason?: string;
+    code?: string;
+    constructor({ type, reason, code, message, }?: {
+        type?: string;
+        reason?: string;
+        code?: string;
+        message?: string;
+    });
+}
+export interface MultimodalLLMMetrics extends LLMMetrics {
+    inputTokenDetails: {
+        cachedTokens: number;
+        textTokens: number;
+        audioTokens: number;
+    };
+    outputTokenDetails: {
+        textTokens: number;
+        audioTokens: number;
+    };
+}
+export type AgentMetrics = STTMetrics | LLMMetrics | TTSMetrics | VADMetrics | PipelineSTTMetrics | PipelineEOUMetrics | PipelineLLMMetrics | PipelineTTSMetrics | PipelineVADMetrics | MultimodalLLMMetrics;
+//# sourceMappingURL=base.d.ts.map

package/dist/metrics/index.d.cts

@@ -0,0 +1,5 @@
+export type { AgentMetrics, STTMetrics, LLMMetrics, TTSMetrics, VADMetrics, PipelineSTTMetrics, PipelineEOUMetrics, PipelineLLMMetrics, PipelineTTSMetrics, PipelineVADMetrics, MultimodalLLMMetrics, } from './base.js';
+export { MultimodalLLMError } from './base.js';
+export { type UsageSummary, UsageCollector } from './usage_collector.js';
+export { logMetrics } from './utils.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/metrics/usage_collector.d.cts

@@ -0,0 +1,14 @@
+import type { AgentMetrics } from './base.js';
+export interface UsageSummary {
+    llmPromptTokens: number;
+    llmCompletionTokens: number;
+    ttsCharactersCount: number;
+    sttAudioDuration: number;
+}
+export declare class UsageCollector {
+    #private;
+    constructor();
+    collect(metrics: AgentMetrics): void;
+    get summary(): UsageSummary;
+}
+//# sourceMappingURL=usage_collector.d.ts.map

package/dist/metrics/utils.d.cts

@@ -0,0 +1,10 @@
+import type { AgentMetrics, LLMMetrics, PipelineEOUMetrics, PipelineLLMMetrics, PipelineTTSMetrics, STTMetrics, TTSMetrics, VADMetrics } from './base.js';
+export declare const logMetrics: (metrics: AgentMetrics) => void;
+export declare const isLLMMetrics: (metrics: AgentMetrics) => metrics is LLMMetrics;
+export declare const isPipelineLLMMetrics: (metrics: AgentMetrics) => metrics is PipelineLLMMetrics;
+export declare const isVADMetrics: (metrics: AgentMetrics) => metrics is VADMetrics;
+export declare const isPipelineEOUMetrics: (metrics: AgentMetrics) => metrics is PipelineEOUMetrics;
+export declare const isTTSMetrics: (metrics: AgentMetrics) => metrics is TTSMetrics;
+export declare const isPipelineTTSMetrics: (metrics: AgentMetrics) => metrics is PipelineTTSMetrics;
+export declare const isSTTMetrics: (metrics: AgentMetrics) => metrics is STTMetrics;
+//# sourceMappingURL=utils.d.ts.map

package/dist/multimodal/agent_playout.d.cts

@@ -0,0 +1,34 @@
+/// <reference types="node" resolution-mode="require"/>
+import type { AudioFrame } from '@livekit/rtc-node';
+import { type AudioSource } from '@livekit/rtc-node';
+import { EventEmitter } from 'node:events';
+import type { TextAudioSynchronizer } from '../transcription.js';
+import { type AsyncIterableQueue, Future } from '../utils.js';
+export declare const proto: {};
+export declare class PlayoutHandle extends EventEmitter {
+    #private;
+    /** @internal */
+    synchronizer: TextAudioSynchronizer;
+    /** @internal */
+    doneFut: Future;
+    /** @internal */
+    intFut: Future;
+    /** @internal */
+    pushedDuration: number;
+    /** @internal */
+    totalPlayedTime: number | undefined;
+    constructor(audioSource: AudioSource, sampleRate: number, itemId: string, contentIndex: number, synchronizer: TextAudioSynchronizer);
+    get itemId(): string;
+    get audioSamples(): number;
+    get textChars(): number;
+    get contentIndex(): number;
+    get interrupted(): boolean;
+    get done(): boolean;
+    interrupt(): void;
+}
+export declare class AgentPlayout extends EventEmitter {
+    #private;
+    constructor(audioSource: AudioSource, sampleRate: number, numChannels: number, inFrameSize: number, outFrameSize: number);
+    play(itemId: string, contentIndex: number, synchronizer: TextAudioSynchronizer, textStream: AsyncIterableQueue<string>, audioStream: AsyncIterableQueue<AudioFrame>): PlayoutHandle;
+}
+//# sourceMappingURL=agent_playout.d.ts.map

package/dist/multimodal/index.d.cts

@@ -0,0 +1,3 @@
+export * from './multimodal_agent.js';
+export * from './agent_playout.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/multimodal/multimodal_agent.d.cts

@@ -0,0 +1,48 @@
+/// <reference types="node" resolution-mode="require"/>
+import type { NoiseCancellationOptions, RemoteAudioTrack, RemoteParticipant, Room } from '@livekit/rtc-node';
+import { EventEmitter } from 'node:events';
+import * as llm from '../llm/index.js';
+/**
+ * @internal
+ * @beta
+ */
+export declare abstract class RealtimeSession extends EventEmitter {
+    abstract conversation: any;
+    abstract inputAudioBuffer: any;
+    abstract fncCtx: llm.FunctionContext | undefined;
+    abstract recoverFromTextResponse(itemId: string): void;
+}
+/**
+ * @internal
+ * @beta
+ */
+export declare abstract class RealtimeModel {
+    abstract session(options: any): RealtimeSession;
+    abstract close(): Promise<void>;
+    abstract sampleRate: number;
+    abstract numChannels: number;
+    abstract inFrameSize: number;
+    abstract outFrameSize: number;
+}
+export type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';
+export declare const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
+/** @beta */
+export declare class MultimodalAgent extends EventEmitter {
+    #private;
+    model: RealtimeModel;
+    room: Room | null;
+    linkedParticipant: RemoteParticipant | null;
+    subscribedTrack: RemoteAudioTrack | null;
+    readMicroTask: Promise<void> | null;
+    constructor({ model, chatCtx, fncCtx, maxTextResponseRetries, noiseCancellation, }: {
+        model: RealtimeModel;
+        chatCtx?: llm.ChatContext;
+        fncCtx?: llm.FunctionContext;
+        maxTextResponseRetries?: number;
+        noiseCancellation?: NoiseCancellationOptions;
+    });
+    get fncCtx(): llm.FunctionContext | undefined;
+    set fncCtx(ctx: llm.FunctionContext | undefined);
+    start(room: Room, participant?: RemoteParticipant | string | null): Promise<RealtimeSession>;
+}
+//# sourceMappingURL=multimodal_agent.d.ts.map

package/dist/pipeline/agent_output.d.cts

@@ -0,0 +1,33 @@
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { TextAudioSynchronizer } from '../transcription.js';
+import { type TTS } from '../tts/index.js';
+import { AsyncIterableQueue, Future } from '../utils.js';
+import type { AgentPlayout, PlayoutHandle } from './agent_playout.js';
+export type SpeechSource = AsyncIterable<string> | string | Promise<string>;
+export declare class SynthesisHandle {
+    #private;
+    static readonly FLUSH_SENTINEL: unique symbol;
+    text?: string;
+    ttsSource: SpeechSource;
+    tts: TTS;
+    queue: AsyncIterableQueue<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
+    intFut: Future;
+    synchronizer: TextAudioSynchronizer;
+    constructor(speechId: string, ttsSource: SpeechSource, agentPlayout: AgentPlayout, tts: TTS, synchronizer: TextAudioSynchronizer);
+    get speechId(): string;
+    get validated(): boolean;
+    get interrupted(): boolean;
+    get playHandle(): PlayoutHandle | undefined;
+    /** Validate the speech for playout. */
+    play(): PlayoutHandle;
+    /** Interrupt the speech. */
+    interrupt(): void;
+}
+export declare class AgentOutput {
+    #private;
+    constructor(agentPlayout: AgentPlayout, tts: TTS);
+    get playout(): AgentPlayout;
+    close(): Promise<void>;
+    synthesize(speechId: string, ttsSource: SpeechSource, synchronizer: TextAudioSynchronizer): SynthesisHandle;
+}
+//# sourceMappingURL=agent_output.d.ts.map

package/dist/pipeline/agent_playout.d.cts

@@ -0,0 +1,40 @@
+import type { AudioFrame, AudioSource } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { TextAudioSynchronizer } from '../transcription.js';
+import { Future } from '../utils.js';
+import { SynthesisHandle } from './agent_output.js';
+export declare enum AgentPlayoutEvent {
+    PLAYOUT_STARTED = 0,
+    PLAYOUT_STOPPED = 1
+}
+export type AgentPlayoutCallbacks = {
+    [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;
+    [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;
+};
+export declare class PlayoutHandle {
+    #private;
+    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
+    totalPlayedTime?: number;
+    synchronizer: TextAudioSynchronizer;
+    pushedDuration: number;
+    intFut: Future;
+    doneFut: Future;
+    constructor(speechId: string, audioSource: AudioSource, playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>, synchronizer: TextAudioSynchronizer);
+    get speechId(): string;
+    get interrupted(): boolean;
+    get timePlayed(): number;
+    get done(): boolean;
+    interrupt(): void;
+    join(): Future;
+}
+declare const AgentPlayout_base: new () => TypedEmitter<AgentPlayoutCallbacks>;
+export declare class AgentPlayout extends AgentPlayout_base {
+    #private;
+    constructor(audioSource: AudioSource);
+    get targetVolume(): number;
+    set targetVolume(vol: number);
+    play(speechId: string, playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>, synchronizer: TextAudioSynchronizer): PlayoutHandle;
+    close(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=agent_playout.d.ts.map

package/dist/pipeline/human_input.d.cts

@@ -0,0 +1,30 @@
+import type { NoiseCancellationOptions, RemoteAudioTrack, RemoteParticipant, Room } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { STT, SpeechEvent } from '../stt/stt.js';
+import type { VAD, VADEvent } from '../vad.js';
+export declare enum HumanInputEvent {
+    START_OF_SPEECH = 0,
+    VAD_INFERENCE_DONE = 1,
+    END_OF_SPEECH = 2,
+    FINAL_TRANSCRIPT = 3,
+    INTERIM_TRANSCRIPT = 4
+}
+export type HumanInputCallbacks = {
+    [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;
+    [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;
+    [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;
+    [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;
+    [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;
+};
+declare const HumanInput_base: new () => TypedEmitter<HumanInputCallbacks>;
+export declare class HumanInput extends HumanInput_base {
+    #private;
+    constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant, noiseCancellation?: NoiseCancellationOptions);
+    get participant(): RemoteParticipant;
+    get subscribedTrack(): RemoteAudioTrack | undefined;
+    get speaking(): boolean;
+    get speakingProbability(): number;
+    close(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=human_input.d.ts.map

package/dist/pipeline/index.d.cts

@@ -0,0 +1,2 @@
+export { type AgentState, type BeforeTTSCallback, type BeforeLLMCallback, type VPACallbacks, type AgentTranscriptionOptions, type VPAOptions, VPAEvent, VoicePipelineAgent, AgentCallContext, } from './pipeline_agent.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/pipeline/pipeline_agent.d.cts

@@ -0,0 +1,151 @@
+import type { NoiseCancellationOptions, RemoteParticipant, Room } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { CallableFunctionResult, FunctionCallInfo, FunctionContext, LLM } from '../llm/index.js';
+import { LLMStream } from '../llm/index.js';
+import { ChatContext, ChatMessage } from '../llm/index.js';
+import type { AgentMetrics } from '../metrics/base.js';
+import { type STT } from '../stt/index.js';
+import type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';
+import type { TTS } from '../tts/index.js';
+import { type VAD } from '../vad.js';
+import type { SpeechSource } from './agent_output.js';
+import { SpeechHandle } from './speech_handle.js';
+export type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';
+export declare const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
+export type BeforeLLMCallback = (agent: VoicePipelineAgent, chatCtx: ChatContext) => LLMStream | false | void | Promise<LLMStream | false | void>;
+export type BeforeTTSCallback = (agent: VoicePipelineAgent, source: string | AsyncIterable<string>) => SpeechSource;
+export declare enum VPAEvent {
+    USER_STARTED_SPEAKING = 0,
+    USER_STOPPED_SPEAKING = 1,
+    AGENT_STARTED_SPEAKING = 2,
+    AGENT_STOPPED_SPEAKING = 3,
+    USER_SPEECH_COMMITTED = 4,
+    AGENT_SPEECH_COMMITTED = 5,
+    AGENT_SPEECH_INTERRUPTED = 6,
+    FUNCTION_CALLS_COLLECTED = 7,
+    FUNCTION_CALLS_FINISHED = 8,
+    METRICS_COLLECTED = 9
+}
+export type VPACallbacks = {
+    [VPAEvent.USER_STARTED_SPEAKING]: () => void;
+    [VPAEvent.USER_STOPPED_SPEAKING]: () => void;
+    [VPAEvent.AGENT_STARTED_SPEAKING]: () => void;
+    [VPAEvent.AGENT_STOPPED_SPEAKING]: () => void;
+    [VPAEvent.USER_SPEECH_COMMITTED]: (msg: ChatMessage) => void;
+    [VPAEvent.AGENT_SPEECH_COMMITTED]: (msg: ChatMessage) => void;
+    [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;
+    [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;
+    [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;
+    [VPAEvent.METRICS_COLLECTED]: (metrics: AgentMetrics) => void;
+};
+interface TurnDetector {
+    unlikelyThreshold: number;
+    supportsLanguage: (language?: string) => boolean;
+    predictEndOfTurn: (chatCtx: ChatContext) => Promise<number>;
+}
+export declare class AgentCallContext {
+    #private;
+    constructor(agent: VoicePipelineAgent, llmStream: LLMStream);
+    static getCurrent(): AgentCallContext;
+    get agent(): VoicePipelineAgent;
+    storeMetadata(key: string, value: any): void;
+    getMetadata(key: string, orDefault?: any): any;
+    get llmStream(): LLMStream;
+    get extraChatMessages(): ChatMessage[];
+    addExtraChatMessage(message: ChatMessage): void;
+}
+export interface AgentTranscriptionOptions {
+    /** Whether to forward the user transcription to the client */
+    userTranscription: boolean;
+    /** Whether to forward the agent transcription to the client */
+    agentTranscription: boolean;
+    /**
+     * The speed at which the agent's speech transcription is forwarded to the client.
+     * We try to mimic the agent's speech speed by adjusting the transcription speed.
+     */
+    agentTranscriptionSpeech: number;
+    /**
+     * The tokenizer used to split the speech into sentences.
+     * This is used to decide when to mark a transcript as final for the agent transcription.
+     */
+    sentenceTokenizer: SentenceTokenizer;
+    /**
+     * The tokenizer used to split the speech into words.
+     * This is used to simulate the "interim results" of the agent transcription.
+     */
+    wordTokenizer: WordTokenizer;
+    /**
+     * A function that takes a string (word) as input and returns a list of strings,
+     * representing the hyphenated parts of the word.
+     */
+    hyphenateWord: (word: string) => string[];
+}
+export interface VPAOptions {
+    /** Chat context for the assistant. */
+    chatCtx?: ChatContext;
+    /** Function context for the assistant. */
+    fncCtx?: FunctionContext;
+    /** Whether to allow the user to interrupt the assistant. */
+    allowInterruptions: boolean;
+    /** Minimum duration of speech to consider for interruption. */
+    interruptSpeechDuration: number;
+    /** Minimum number of words to consider for interuption. This may increase latency. */
+    interruptMinWords: number;
+    /** Delay to wait before considering the user speech done. */
+    minEndpointingDelay: number;
+    maxNestedFncCalls: number;
+    preemptiveSynthesis: boolean;
+    beforeLLMCallback: BeforeLLMCallback;
+    beforeTTSCallback: BeforeTTSCallback;
+    /** Options for assistant transcription. */
+    transcription: AgentTranscriptionOptions;
+    /** Turn detection model to use. */
+    turnDetector?: TurnDetector;
+    /** Noise cancellation options. */
+    noiseCancellation?: NoiseCancellationOptions;
+}
+declare const VoicePipelineAgent_base: new () => TypedEmitter<VPACallbacks>;
+/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */
+export declare class VoicePipelineAgent extends VoicePipelineAgent_base {
+    #private;
+    /** Minimum time played for the user speech to be committed to the chat context. */
+    readonly MIN_TIME_PLAYED_FOR_COMMIT = 1.5;
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    transcribedText: string;
+    constructor(
+    /** Voice Activity Detection instance. */
+    vad: VAD, 
+    /** Speech-to-Text instance. */
+    stt: STT, 
+    /** Large Language Model instance. */
+    llm: LLM, 
+    /** Text-to-Speech instance. */
+    tts: TTS, 
+    /** Additional VoicePipelineAgent options. */
+    opts?: Partial<VPAOptions>);
+    get fncCtx(): FunctionContext | undefined;
+    set fncCtx(ctx: FunctionContext);
+    get chatCtx(): ChatContext;
+    get llm(): LLM;
+    get tts(): TTS;
+    get stt(): STT;
+    get vad(): VAD;
+    /** Start the voice assistant. */
+    start(
+    /** The room to connect to. */
+    room: Room, 
+    /**
+     * The participant to listen to.
+     *
+     * @remarks
+     * Can be a participant or an identity.
+     * If omitted, the first participant in the room will be selected.
+     */
+    participant?: RemoteParticipant | string | null): void;
+    /** Play a speech source through the voice assistant. */
+    say(source: string | LLMStream | AsyncIterable<string>, allowInterruptions?: boolean, addToChatCtx?: boolean): Promise<SpeechHandle>;
+    /** Close the voice assistant. */
+    close(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=pipeline_agent.d.ts.map

package/dist/pipeline/speech_handle.d.cts

@@ -0,0 +1,37 @@
+import type { ChatMessage, LLMStream } from '../llm/index.js';
+import type { SynthesisHandle } from './agent_output.js';
+export declare class SpeechHandle {
+    #private;
+    constructor(id: string, allowInterruptions: boolean, addToChatCtx: boolean, isReply: boolean, userQuestion: string, fncNestedDepth?: number, extraToolsMessages?: ChatMessage[] | undefined);
+    static createAssistantReply(allowInterruptions: boolean, addToChatCtx: boolean, userQuestion: string): SpeechHandle;
+    static createAssistantSpeech(allowInterruptions: boolean, addToChatCtx: boolean): SpeechHandle;
+    static createToolSpeech(allowInterruptions: boolean, addToChatCtx: boolean, fncNestedDepth: number, extraToolsMessages: ChatMessage[]): SpeechHandle;
+    waitForInitialization(): Promise<void>;
+    initialize(source: string | LLMStream | AsyncIterable<string>, synthesisHandle: SynthesisHandle): void;
+    markUserCommitted(): void;
+    markSpeechCommitted(): void;
+    get userCommitted(): boolean;
+    get speechCommitted(): boolean;
+    get id(): string;
+    get allowInterruptions(): boolean;
+    get addToChatCtx(): boolean;
+    get source(): string | LLMStream | AsyncIterable<string>;
+    get synthesisHandle(): SynthesisHandle;
+    set synthesisHandle(handle: SynthesisHandle);
+    get initialized(): boolean;
+    get isReply(): boolean;
+    get userQuestion(): string;
+    get interrupted(): boolean;
+    get fncNestedDepth(): number;
+    get extraToolsMessages(): ChatMessage[] | undefined;
+    addNestedSpeech(handle: SpeechHandle): void;
+    get nestedSpeechHandles(): SpeechHandle[];
+    nestedSpeechChanged(): Promise<void>;
+    get nestedSpeechFinished(): boolean;
+    markNestedSpeechFinished(): void;
+    join(): Promise<void>;
+    setDone(): void;
+    interrupt(): void;
+    cancel(): void;
+}
+//# sourceMappingURL=speech_handle.d.ts.map

package/dist/plugin.d.cts

@@ -0,0 +1,10 @@
+export declare abstract class Plugin {
+    #private;
+    registeredPlugins: Plugin[];
+    constructor(title: string, version: string);
+    static registerPlugins(plugin: Plugin): void;
+    abstract downloadFiles(): void;
+    get title(): string;
+    get version(): string;
+}
+//# sourceMappingURL=plugin.d.ts.map

package/dist/stt/index.d.cts

@@ -0,0 +1,3 @@
+export { type SpeechEvent, type SpeechData, type STTCapabilities, type RecognitionUsage, type STTCallbacks, SpeechEventType, STT, SpeechStream, } from './stt.js';
+export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/stt/stream_adapter.d.cts

@@ -0,0 +1,18 @@
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { VAD } from '../vad.js';
+import type { SpeechEvent } from './stt.js';
+import { STT, SpeechStream } from './stt.js';
+export declare class StreamAdapter extends STT {
+    #private;
+    label: string;
+    constructor(stt: STT, vad: VAD);
+    _recognize(frame: AudioFrame): Promise<SpeechEvent>;
+    stream(): StreamAdapterWrapper;
+}
+export declare class StreamAdapterWrapper extends SpeechStream {
+    #private;
+    label: string;
+    constructor(stt: STT, vad: VAD);
+    monitorMetrics(): Promise<void>;
+}
+//# sourceMappingURL=stream_adapter.d.ts.map