@livekit/agents 0.3.4 → 0.4.0

package/dist/pipeline/pipeline_agent.d.ts

@@ -0,0 +1,134 @@
+import type { RemoteParticipant, Room } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { CallableFunctionResult, FunctionCallInfo, FunctionContext, LLM } from '../llm/index.js';
+import { LLMStream } from '../llm/index.js';
+import { ChatContext, ChatMessage } from '../llm/index.js';
+import type { STT } from '../stt/index.js';
+import type { SentenceTokenizer, WordTokenizer } from '../tokenize/tokenizer.js';
+import type { TTS } from '../tts/index.js';
+import type { VAD } from '../vad.js';
+import type { SpeechSource } from './agent_output.js';
+export type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';
+export type BeforeLLMCallback = (agent: VoicePipelineAgent, chatCtx: ChatContext) => LLMStream | false | void | Promise<LLMStream | false | void>;
+export type BeforeTTSCallback = (agent: VoicePipelineAgent, source: string | AsyncIterable<string>) => SpeechSource;
+export declare enum VPAEvent {
+    USER_STARTED_SPEAKING = 0,
+    USER_STOPPED_SPEAKING = 1,
+    AGENT_STARTED_SPEAKING = 2,
+    AGENT_STOPPED_SPEAKING = 3,
+    USER_SPEECH_COMMITTED = 4,
+    AGENT_SPEECH_COMMITTED = 5,
+    AGENT_SPEECH_INTERRUPTED = 6,
+    FUNCTION_CALLS_COLLECTED = 7,
+    FUNCTION_CALLS_FINISHED = 8
+}
+export type VPACallbacks = {
+    [VPAEvent.USER_STARTED_SPEAKING]: () => void;
+    [VPAEvent.USER_STOPPED_SPEAKING]: () => void;
+    [VPAEvent.AGENT_STARTED_SPEAKING]: () => void;
+    [VPAEvent.AGENT_STOPPED_SPEAKING]: () => void;
+    [VPAEvent.USER_SPEECH_COMMITTED]: (msg: ChatMessage) => void;
+    [VPAEvent.AGENT_SPEECH_COMMITTED]: (msg: ChatMessage) => void;
+    [VPAEvent.AGENT_SPEECH_INTERRUPTED]: (msg: ChatMessage) => void;
+    [VPAEvent.FUNCTION_CALLS_COLLECTED]: (funcs: FunctionCallInfo[]) => void;
+    [VPAEvent.FUNCTION_CALLS_FINISHED]: (funcs: CallableFunctionResult[]) => void;
+};
+export declare class AgentCallContext {
+    #private;
+    constructor(agent: VoicePipelineAgent, llmStream: LLMStream);
+    static getCurrent(): AgentCallContext;
+    get agent(): VoicePipelineAgent;
+    storeMetadata(key: string, value: any): void;
+    getMetadata(key: string, orDefault?: any): any;
+    get llmStream(): LLMStream;
+}
+export interface AgentTranscriptionOptions {
+    /** Whether to forward the user transcription to the client */
+    userTranscription: boolean;
+    /** Whether to forward the agent transcription to the client */
+    agentTranscription: boolean;
+    /**
+     * The speed at which the agent's speech transcription is forwarded to the client.
+     * We try to mimic the agent's speech speed by adjusting the transcription speed.
+     */
+    agentTranscriptionSpeech: number;
+    /**
+     * The tokenizer used to split the speech into sentences.
+     * This is used to decide when to mark a transcript as final for the agent transcription.
+     */
+    sentenceTokenizer: SentenceTokenizer;
+    /**
+     * The tokenizer used to split the speech into words.
+     * This is used to simulate the "interim results" of the agent transcription.
+     */
+    wordTokenizer: WordTokenizer;
+    /**
+     * A function that takes a string (word) as input and returns a list of strings,
+     * representing the hyphenated parts of the word.
+     */
+    hyphenateWord: (word: string) => string[];
+}
+export interface VPAOptions {
+    /** Chat context for the assistant. */
+    chatCtx?: ChatContext;
+    /** Function context for the assistant. */
+    fncCtx?: FunctionContext;
+    /** Whether to allow the user to interrupt the assistant. */
+    allowInterruptions: boolean;
+    /** Minimum duration of speech to consider for interruption. */
+    interruptSpeechDuration: number;
+    /** Minimum number of words to consider for interuption. This may increase latency. */
+    interruptMinWords: number;
+    /** Delay to wait before considering the user speech done. */
+    minEndpointingDelay: number;
+    maxRecursiveFncCalls: number;
+    preemptiveSynthesis: boolean;
+    beforeLLMCallback: BeforeLLMCallback;
+    beforeTTSCallback: BeforeTTSCallback;
+    /** Options for assistant transcription. */
+    transcription: AgentTranscriptionOptions;
+}
+declare const VoicePipelineAgent_base: new () => TypedEmitter<VPACallbacks>;
+/** A pipeline agent (VAD + STT + LLM + TTS) implementation. */
+export declare class VoicePipelineAgent extends VoicePipelineAgent_base {
+    #private;
+    /** Minimum time played for the user speech to be committed to the chat context. */
+    readonly MIN_TIME_PLAYED_FOR_COMMIT = 1.5;
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    constructor(
+    /** Voice Activity Detection instance. */
+    vad: VAD, 
+    /** Speech-to-Text instance. */
+    stt: STT, 
+    /** Large Language Model instance. */
+    llm: LLM, 
+    /** Text-to-Speech instance. */
+    tts: TTS, 
+    /** Additional VoicePipelineAgent options. */
+    opts?: Partial<VPAOptions>);
+    get fncCtx(): FunctionContext | undefined;
+    set fncCtx(ctx: FunctionContext);
+    get chatCtx(): ChatContext;
+    get llm(): LLM;
+    get tts(): TTS;
+    get stt(): STT;
+    get vad(): VAD;
+    /** Start the voice assistant. */
+    start(
+    /** The room to connect to. */
+    room: Room, 
+    /**
+     * The participant to listen to.
+     *
+     * @remarks
+     * Can be a participant or an identity.
+     * If omitted, the first participant in the room will be selected.
+     */
+    participant?: RemoteParticipant | string | null): void;
+    /** Play a speech source through the voice assistant. */
+    say(source: string | LLMStream | AsyncIterable<string>, allowInterruptions?: boolean, addToChatCtx?: boolean): Promise<void>;
+    /** Close the voice assistant. */
+    close(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=pipeline_agent.d.ts.map

package/dist/pipeline/pipeline_agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline_agent.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline_agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAyB,iBAAiB,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAQxF,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EACV,sBAAsB,EACtB,gBAAgB,EAChB,eAAe,EACf,GAAG,EACJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAY,MAAM,iBAAiB,CAAC;AAErE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAM3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACjF,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAY,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,YAAY,EAAmB,MAAM,mBAAmB,CAAC;AAMvE,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAEhF,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE,WAAW,KACjB,SAAS,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;AAElE,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,kBAAkB,EACzB,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KACnC,YAAY,CAAC;AAElB,oBAAY,QAAQ;IAClB,qBAAqB,IAAA;IACrB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,sBAAsB,IAAA;IACtB,qBAAqB,IAAA;IACrB,sBAAsB,IAAA;IACtB,wBAAwB,IAAA;IACxB,wBAAwB,IAAA;IACxB,uBAAuB,IAAA;CACxB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC7C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9C,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7D,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAC9D,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,IAAI,CAAC;IAChE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;IACzE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,IAAI,CAAC;CAC/E,CAAC;AAEF,qBAAa,gBAAgB;;gBAMf,KAAK,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS;IAM3D,MAAM,CAAC,UAAU,IAAI,gBAAgB;IAIrC,IAAI,KAAK,IAAI,kBAAkB,CAE9B;IAED,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG;IAIrC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,GAAe;IAInD,IAAI,SAAS,IAAI,SAAS,CAEzB;CACF;AAiBD,MAAM,WAAW,yBAAyB;IACxC,8DAA8D;IAC9D,iBAAiB,EAAE,OAAO,CAAC;IAC3B,+DAA+D;IAC/D,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,wBAAwB,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,aAAa,EAAE,aAAa,CAAC;IAC7B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC3C;AAWD,MAAM,WAAW,UAAU;IACzB,sCAAsC;IACtC,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,4DAA4D;IAC5D,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+DAA+D;IAC/D,uBAAuB,EAAE,MAAM,CAAC;IAChC,sFAAsF;IACtF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6DAA6D;IAC7D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oBAAoB,EAAE,MAAM,CAAC;IAE7B,mBAAmB,EAAE,OAAO,CAAC;IAS7B,iBAAiB,EAAE,iBAAiB,CAAC;IAQrC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,2CAA2C;IAC3C,aAAa,EAAE,yBAAyB,CAAC;CAC1C;iDAgBkE,aAAa,YAAY,CAAC;AAD7F,+DAA+D;AAC/D,qBAAa,kBAAmB,SAAQ,uBAAsD;;IAC5F,mFAAmF;IACnF,QAAQ,CAAC,0BAA0B,OAAO;IAC1C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;;IA2BlE,yCAAyC;IACzC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,qCAAqC;IACrC,GAAG,EAAE,GAAG;IACR,+BAA+B;IAC/B,GAAG,EAAE,GAAG;IACR,6CAA6C;IAC7C,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAiB/C,IAAI,MAAM,IAAI,eAAe,GAAG,SAAS,CAExC;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,eAAe,EAE9B;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,IAAI,GAAG,IAAI,GAAG,CAEb;IAED,iCAAiC;IACjC,KAAK;IACH,8BAA8B;IAC9B,IAAI,EAAE,IAAI;IACV;;;;;;OAMG;IACH,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW;IA2BvD,wDAAwD;IAClD,GAAG,CACP,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,EAClD,kBAAkB,UAAO,EACzB,YAAY,UAAO;IAkdrB,iCAAiC;IAC3B,KAAK;CAQZ"}