@livekit/agents 0.7.9 → 1.0.0-next.0

package/src/pipeline/agent_output.ts

@@ -1,219 +0,0 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import type { AudioFrame } from '@livekit/rtc-node';
-import { log } from '../log.js';
-import type { TextAudioSynchronizer } from '../transcription.js';
-import { SynthesizeStream, type TTS } from '../tts/index.js';
-import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
-import type { AgentPlayout, PlayoutHandle } from './agent_playout.js';
-
-export type SpeechSource = AsyncIterable<string> | string | Promise<string>;
-
-export class SynthesisHandle {
-  static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
-
-  #speechId: string;
-  text?: string;
-  ttsSource: SpeechSource;
-  #agentPlayout: AgentPlayout;
-  tts: TTS;
-  queue = new AsyncIterableQueue<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>();
-  #playHandle?: PlayoutHandle;
-  intFut = new Future();
-  #logger = log();
-  synchronizer: TextAudioSynchronizer;
-
-  constructor(
-    speechId: string,
-    ttsSource: SpeechSource,
-    agentPlayout: AgentPlayout,
-    tts: TTS,
-    synchronizer: TextAudioSynchronizer,
-  ) {
-    this.#speechId = speechId;
-    this.ttsSource = ttsSource;
-    this.#agentPlayout = agentPlayout;
-    this.tts = tts;
-    this.synchronizer = synchronizer;
-  }
-
-  get speechId(): string {
-    return this.#speechId;
-  }
-
-  get validated(): boolean {
-    return !!this.#playHandle;
-  }
-
-  get interrupted(): boolean {
-    return this.intFut.done;
-  }
-
-  get playHandle(): PlayoutHandle | undefined {
-    return this.#playHandle;
-  }
-
-  /** Validate the speech for playout. */
-  play(): PlayoutHandle {
-    if (this.interrupted) {
-      throw new Error('synthesis was interrupted');
-    }
-
-    this.#playHandle = this.#agentPlayout.play(this.#speechId, this.queue, this.synchronizer);
-    return this.#playHandle;
-  }
-
-  /** Interrupt the speech. */
-  interrupt() {
-    if (this.interrupted) {
-      return;
-    }
-
-    this.#logger.child({ speechId: this.#speechId }).debug('interrupting synthesis/playout');
-    this.#playHandle?.interrupt();
-    this.intFut.resolve();
-  }
-}
-
-export class AgentOutput {
-  #agentPlayout: AgentPlayout;
-  #tts: TTS;
-  #tasks: CancellablePromise<void>[] = [];
-
-  constructor(agentPlayout: AgentPlayout, tts: TTS) {
-    this.#agentPlayout = agentPlayout;
-    this.#tts = tts;
-  }
-
-  get playout(): AgentPlayout {
-    return this.#agentPlayout;
-  }
-
-  async close() {
-    this.#tasks.forEach((task) => task.cancel());
-    await Promise.all(this.#tasks);
-  }
-
-  synthesize(
-    speechId: string,
-    ttsSource: SpeechSource,
-    synchronizer: TextAudioSynchronizer,
-  ): SynthesisHandle {
-    const handle = new SynthesisHandle(
-      speechId,
-      ttsSource,
-      this.#agentPlayout,
-      this.#tts,
-      synchronizer,
-    );
-    const task = this.#synthesize(handle);
-    this.#tasks.push(task);
-    task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
-    return handle;
-  }
-
-  #synthesize(handle: SynthesisHandle): CancellablePromise<void> {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    return new CancellablePromise(async (resolve, _, onCancel) => {
-      const ttsSource = await handle.ttsSource;
-      let task: CancellablePromise<string>;
-      if (typeof ttsSource === 'string') {
-        task = stringSynthesisTask(ttsSource, handle);
-      } else {
-        task = streamSynthesisTask(ttsSource, handle);
-      }
-
-      onCancel(() => {
-        gracefullyCancel(task);
-      });
-
-      try {
-        await Promise.any([task, handle.intFut.await]);
-      } finally {
-        if (handle.intFut.done) {
-          gracefullyCancel(task);
-        } else {
-          task.then((text) => {
-            handle.text = text;
-          });
-        }
-      }
-
-      resolve();
-    });
-  }
-}
-
-const stringSynthesisTask = (text: string, handle: SynthesisHandle): CancellablePromise<string> => {
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  return new CancellablePromise(async (resolve, _, onCancel) => {
-    let cancelled = false;
-    onCancel(() => {
-      cancelled = true;
-    });
-
-    const ttsStream = handle.tts.stream();
-    ttsStream.pushText(text);
-    handle.synchronizer.pushText(text);
-    handle.synchronizer.markTextSegmentEnd();
-    ttsStream.flush();
-    ttsStream.endInput();
-    for await (const audio of ttsStream) {
-      if (cancelled || audio === SynthesizeStream.END_OF_STREAM) {
-        break;
-      }
-      handle.queue.put(audio.frame);
-    }
-    handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
-
-    resolve(text);
-  });
-};
-
-const streamSynthesisTask = (
-  stream: AsyncIterable<string>,
-  handle: SynthesisHandle,
-): CancellablePromise<string> => {
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  return new CancellablePromise(async (resolve, _, onCancel) => {
-    let fullText = '';
-    let cancelled = false;
-    onCancel(() => {
-      cancelled = true;
-    });
-
-    const ttsStream = handle.tts.stream();
-    const readGeneratedAudio = async () => {
-      for await (const audio of ttsStream) {
-        if (cancelled) break;
-        if (audio === SynthesizeStream.END_OF_STREAM) {
-          break;
-        }
-        handle.queue.put(audio.frame);
-      }
-      handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
-    };
-    readGeneratedAudio();
-
-    for await (const text of stream) {
-      fullText += text;
-      if (cancelled) break;
-      handle.synchronizer.pushText(text);
-      ttsStream.pushText(text);
-    }
-    if (!cancelled) {
-      handle.synchronizer.markTextSegmentEnd();
-    }
-
-    // end the audio queue early if there is no actual text to turn into speech
-    if (!fullText || fullText.trim().length === 0) {
-      cancelled = true;
-      handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
-    }
-    ttsStream.flush();
-    ttsStream.endInput();
-
-    resolve(fullText);
-  });
-};

package/src/pipeline/agent_playout.ts

@@ -1,192 +0,0 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import type { AudioFrame, AudioSource } from '@livekit/rtc-node';
-import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
-import EventEmitter from 'node:events';
-import { log } from '../log.js';
-import type { TextAudioSynchronizer } from '../transcription.js';
-import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';
-import { SynthesisHandle } from './agent_output.js';
-
-export enum AgentPlayoutEvent {
-  PLAYOUT_STARTED,
-  PLAYOUT_STOPPED,
-}
-
-export type AgentPlayoutCallbacks = {
-  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;
-  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;
-};
-
-export class PlayoutHandle {
-  #speechId: string;
-  #audioSource: AudioSource;
-  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
-  totalPlayedTime?: number;
-  synchronizer: TextAudioSynchronizer;
-  #interrupted = false;
-  pushedDuration = 0;
-  intFut = new Future();
-  doneFut = new Future();
-
-  constructor(
-    speechId: string,
-    audioSource: AudioSource,
-    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
-    synchronizer: TextAudioSynchronizer,
-  ) {
-    this.#speechId = speechId;
-    this.#audioSource = audioSource;
-    this.playoutSource = playoutSource;
-    this.synchronizer = synchronizer;
-  }
-
-  get speechId(): string {
-    return this.#speechId;
-  }
-
-  get interrupted(): boolean {
-    return this.#interrupted;
-  }
-
-  get timePlayed(): number {
-    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;
-  }
-
-  get done(): boolean {
-    return this.doneFut.done || this.#interrupted;
-  }
-
-  interrupt() {
-    if (this.done) {
-      return;
-    }
-
-    this.intFut.resolve();
-    this.#interrupted = true;
-  }
-
-  join(): Future {
-    return this.doneFut;
-  }
-}
-
-export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {
-  #closed = false;
-  #audioSource: AudioSource;
-  #targetVolume = 1;
-  #playoutTask?: CancellablePromise<void>;
-  #logger = log();
-
-  constructor(audioSource: AudioSource) {
-    super();
-    this.#audioSource = audioSource;
-  }
-
-  get targetVolume(): number {
-    return this.#targetVolume;
-  }
-
-  set targetVolume(vol: number) {
-    this.#targetVolume = vol;
-  }
-
-  play(
-    speechId: string,
-    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
-    synchronizer: TextAudioSynchronizer,
-  ): PlayoutHandle {
-    if (this.#closed) {
-      throw new Error('source closed');
-    }
-
-    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource, synchronizer);
-
-    this.#playoutTask = this.#playout(handle, this.#playoutTask);
-    return handle;
-  }
-
-  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {
-    return new CancellablePromise(async (resolve, _, onCancel) => {
-      const cancel = () => {
-        captureTask.cancel();
-        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
-
-        if (handle.interrupted || captureTask.error) {
-          handle.synchronizer.close(true);
-          this.#audioSource.clearQueue(); // make sure to remove any queued frames
-        }
-
-        if (!firstFrame) {
-          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);
-        }
-
-        handle.doneFut.resolve();
-
-        this.#logger
-          .child({ speechId: handle.speechId, interrupted: handle.interrupted })
-          .debug('playout finished');
-      };
-
-      onCancel(() => {
-        cancel();
-      });
-
-      if (oldTask) {
-        await gracefullyCancel(oldTask);
-      }
-
-      if (this.#audioSource.queuedDuration > 0) {
-        // this should not happen, but log it just in case
-        this.#logger
-          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })
-          .warn('new playout while the source is still playing');
-      }
-
-      let firstFrame = true;
-
-      // eslint-disable-next-line @typescript-eslint/no-unused-vars
-      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {
-        let cancelled = false;
-        onCancel(() => {
-          cancelled = true;
-        });
-
-        for await (const frame of handle.playoutSource) {
-          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) {
-            break;
-          }
-          if (firstFrame) {
-            this.#logger
-              .child({ speechId: handle.speechId })
-              .debug('started playing the first time');
-            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);
-            handle.synchronizer.segmentPlayoutStarted();
-            firstFrame = false;
-          }
-          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;
-          handle.synchronizer.pushAudio(frame);
-          await this.#audioSource.captureFrame(frame);
-        }
-
-        await this.#audioSource.waitForPlayout();
-
-        handle.synchronizer.close(false);
-        resolve();
-      });
-
-      try {
-        await Promise.any([captureTask, handle.intFut.await]);
-      } finally {
-        cancel();
-        resolve();
-      }
-    });
-  }
-
-  async close() {
-    this.#closed = true;
-    await this.#playoutTask;
-  }
-}

package/src/pipeline/human_input.ts

@@ -1,188 +0,0 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import type {
-  NoiseCancellationOptions,
-  RemoteAudioTrack,
-  RemoteParticipant,
-  RemoteTrackPublication,
-  Room,
-} from '@livekit/rtc-node';
-import { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';
-import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
-import { EventEmitter } from 'node:events';
-import { log } from '../log.js';
-import type { STT, SpeechEvent } from '../stt/stt.js';
-import { SpeechEventType } from '../stt/stt.js';
-import { CancellablePromise, gracefullyCancel } from '../utils.js';
-import type { VAD, VADEvent } from '../vad.js';
-import { VADEventType } from '../vad.js';
-
-export enum HumanInputEvent {
-  START_OF_SPEECH,
-  VAD_INFERENCE_DONE,
-  END_OF_SPEECH,
-  FINAL_TRANSCRIPT,
-  INTERIM_TRANSCRIPT,
-}
-
-export type HumanInputCallbacks = {
-  [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;
-  [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;
-  [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;
-  [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;
-  [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;
-};
-
-export class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInputCallbacks>) {
-  #closed = false;
-  #room: Room;
-  #vad: VAD;
-  #stt: STT;
-  #participant: RemoteParticipant;
-  #subscribedTrack?: RemoteAudioTrack;
-  #recognizeTask?: CancellablePromise<void>;
-  #speaking = false;
-  #speechProbability = 0;
-  #logger = log();
-  #noiseCancellation?: NoiseCancellationOptions;
-
-  constructor(
-    room: Room,
-    vad: VAD,
-    stt: STT,
-    participant: RemoteParticipant,
-    noiseCancellation?: NoiseCancellationOptions,
-  ) {
-    super();
-    this.#room = room;
-    this.#vad = vad;
-    this.#stt = stt;
-    this.#participant = participant;
-    this.#noiseCancellation = noiseCancellation;
-
-    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));
-    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));
-    this.#subscribeToMicrophone();
-  }
-
-  get participant(): RemoteParticipant {
-    return this.#participant;
-  }
-
-  get subscribedTrack(): RemoteAudioTrack | undefined {
-    return this.#subscribedTrack;
-  }
-
-  #subscribeToMicrophone(): void {
-    if (!this.#participant) {
-      this.#logger.error('Participant is not set');
-      return;
-    }
-
-    let microphonePublication: RemoteTrackPublication | undefined = undefined;
-    for (const publication of this.#participant.trackPublications.values()) {
-      if (publication.source === TrackSource.SOURCE_MICROPHONE) {
-        microphonePublication = publication;
-        break;
-      }
-    }
-    if (!microphonePublication) {
-      return;
-    }
-
-    if (!microphonePublication.subscribed) {
-      microphonePublication.setSubscribed(true);
-    }
-
-    const track = microphonePublication.track;
-    if (track && track !== this.#subscribedTrack) {
-      this.#subscribedTrack = track;
-      if (this.#recognizeTask) {
-        this.#recognizeTask.cancel();
-      }
-
-      const audioStreamOptions = {
-        sampleRate: 16000,
-        numChannels: 1,
-        ...(this.#noiseCancellation ? { noiseCancellation: this.#noiseCancellation } : {}),
-      };
-      const audioStream = new AudioStream(track, audioStreamOptions);
-
-      // eslint-disable-next-line @typescript-eslint/no-unused-vars
-      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {
-        let cancelled = false;
-        onCancel(() => {
-          cancelled = true;
-        });
-
-        const sttStream = this.#stt.stream();
-        const vadStream = this.#vad.stream();
-
-        const audioStreamCo = async () => {
-          for await (const ev of audioStream) {
-            if (cancelled) return;
-            sttStream.pushFrame(ev);
-            vadStream.pushFrame(ev);
-          }
-        };
-
-        const vadStreamCo = async () => {
-          for await (const ev of vadStream) {
-            if (cancelled) return;
-            switch (ev.type) {
-              case VADEventType.START_OF_SPEECH:
-                this.#speaking = true;
-                this.emit(HumanInputEvent.START_OF_SPEECH, ev);
-                break;
-              case VADEventType.INFERENCE_DONE:
-                this.#speechProbability = ev.probability;
-                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);
-                break;
-              case VADEventType.END_OF_SPEECH:
-                this.#speaking = false;
-                this.emit(HumanInputEvent.END_OF_SPEECH, ev);
-                break;
-            }
-          }
-        };
-
-        const sttStreamCo = async () => {
-          for await (const ev of sttStream) {
-            if (cancelled) return;
-            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {
-              this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);
-            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {
-              this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);
-            }
-          }
-        };
-
-        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);
-        sttStream.close();
-        vadStream.close();
-        resolve();
-      });
-    }
-  }
-
-  get speaking(): boolean {
-    return this.#speaking;
-  }
-
-  get speakingProbability(): number {
-    return this.#speechProbability;
-  }
-
-  async close() {
-    if (this.#closed) {
-      throw new Error('HumanInput already closed');
-    }
-    this.#closed = true;
-    this.#room.removeAllListeners();
-    this.#speaking = false;
-    if (this.#recognizeTask) {
-      await gracefullyCancel(this.#recognizeTask);
-    }
-  }
-}

package/src/pipeline/index.ts

@@ -1,15 +0,0 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-export {
-  type AgentState,
-  type BeforeTTSCallback,
-  type BeforeLLMCallback,
-  type VPACallbacks,
-  type AgentTranscriptionOptions,
-  type VPAOptions,
-  VPAEvent,
-  VoicePipelineAgent,
-  AgentCallContext,
-} from './pipeline_agent.js';