@livekit/agents 0.1.0 → 0.3.0

package/src/transcription.ts

@@ -0,0 +1,128 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame, Room } from '@livekit/rtc-node';
+import { log } from './log.js';
+
+export interface TranscriptionForwarder {
+  start(): void;
+  pushAudio(frame: AudioFrame): void;
+  pushText(text: string): void;
+  markTextComplete(): void;
+  markAudioComplete(): void;
+  close(interrupt: boolean): Promise<void>;
+  currentCharacterIndex: number;
+}
+
+export class BasicTranscriptionForwarder implements TranscriptionForwarder {
+  #room: Room;
+  #participantIdentity: string;
+  #trackSid: string;
+  #currentText: string = '';
+  #totalAudioDuration: number = 0;
+  #currentPlayoutTime: number = 0;
+  #DEFAULT_CHARS_PER_SECOND = 16;
+  #charsPerSecond: number = this.#DEFAULT_CHARS_PER_SECOND;
+  #messageId: string;
+  #isRunning: boolean = false;
+  currentCharacterIndex: number = 0;
+
+  constructor(room: Room, participantIdentity: string, trackSid: string, messageId: string) {
+    this.#room = room;
+    this.#participantIdentity = participantIdentity;
+    this.#trackSid = trackSid;
+    this.#messageId = messageId;
+  }
+
+  start(): void {
+    if (!this.#isRunning) {
+      this.#isRunning = true;
+      this.#startPublishingLoop().catch((error) => {
+        log().error('Error in publishing loop:', error);
+        this.#isRunning = false;
+      });
+    }
+  }
+
+  pushAudio(frame: AudioFrame): void {
+    this.#totalAudioDuration += frame.samplesPerChannel / frame.sampleRate;
+  }
+
+  pushText(text: string): void {
+    this.#currentText += text;
+  }
+
+  #textIsComplete: boolean = false;
+  #audioIsComplete: boolean = false;
+
+  markTextComplete(): void {
+    this.#textIsComplete = true;
+    this.#adjustTimingIfBothFinished();
+  }
+
+  markAudioComplete(): void {
+    this.#audioIsComplete = true;
+    this.#adjustTimingIfBothFinished();
+  }
+
+  #adjustTimingIfBothFinished(): void {
+    if (this.#textIsComplete && this.#audioIsComplete) {
+      const actualDuration = this.#totalAudioDuration;
+      if (actualDuration > 0 && this.#currentText.length > 0) {
+        this.#charsPerSecond = this.#currentText.length / actualDuration;
+      }
+    }
+  }
+
+  #computeSleepInterval(): number {
+    return Math.min(Math.max(1 / this.#charsPerSecond, 0.0625), 0.5);
+  }
+
+  async #startPublishingLoop(): Promise<void> {
+    this.#isRunning = true;
+    let sleepInterval = this.#computeSleepInterval();
+    let isComplete = false;
+    while (this.#isRunning && !isComplete) {
+      this.#currentPlayoutTime += sleepInterval;
+      this.currentCharacterIndex = Math.floor(this.#currentPlayoutTime * this.#charsPerSecond);
+      isComplete = this.#textIsComplete && this.currentCharacterIndex >= this.#currentText.length;
+      await this.#publishTranscription(false);
+      if (this.#isRunning && !isComplete) {
+        sleepInterval = this.#computeSleepInterval();
+        await new Promise((resolve) => setTimeout(resolve, sleepInterval * 1000));
+      }
+    }
+
+    if (this.#isRunning) {
+      this.close(false);
+    }
+  }
+
+  async #publishTranscription(final: boolean): Promise<void> {
+    const textToPublish = this.#currentText.slice(0, this.currentCharacterIndex);
+    await this.#room.localParticipant?.publishTranscription({
+      participantIdentity: this.#participantIdentity,
+      trackSid: this.#trackSid,
+      segments: [
+        {
+          text: textToPublish,
+          final: final,
+          id: this.#messageId,
+          startTime: BigInt(0),
+          endTime: BigInt(0),
+          language: '',
+        },
+      ],
+    });
+  }
+
+  async close(interrupt: boolean): Promise<void> {
+    this.#isRunning = false;
+
+    // Publish whatever we had as final
+    if (!interrupt) {
+      this.currentCharacterIndex = this.#currentText.length;
+    }
+    await this.#publishTranscription(true);
+  }
+}

package/src/tts/stream_adapter.ts

@@ -2,25 +2,26 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { SentenceStream, SentenceTokenizer } from '../tokenize.js';
-import { ChunkedStream, SynthesisEvent, SynthesisEventType, SynthesizeStream, TTS } from './tts.js';
+import type { ChunkedStream } from './tts.js';
+import { SynthesisEvent, SynthesisEventType, SynthesizeStream, TTS } from './tts.js';
 
 export class StreamAdapterWrapper extends SynthesizeStream {
-  closed: boolean;
-  tts: TTS;
-  sentenceStream: SentenceStream;
-  eventQueue: (SynthesisEvent | undefined)[];
-  task: {
+  #closed: boolean;
+  #tts: TTS;
+  #sentenceStream: SentenceStream;
+  #eventQueue: (SynthesisEvent | undefined)[];
+  #task: {
     run: Promise<void>;
     cancel: () => void;
   };
 
   constructor(tts: TTS, sentenceStream: SentenceStream) {
     super();
-    this.closed = false;
-    this.tts = tts;
-    this.sentenceStream = sentenceStream;
-    this.eventQueue = [];
-    this.task = {
+    this.#closed = false;
+    this.#tts = tts;
+    this.#sentenceStream = sentenceStream;
+    this.#eventQueue = [];
+    this.#task = {
       run: new Promise((_, reject) => {
         this.run(reject);
       }),
@@ -29,32 +30,32 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   }
 
   async run(reject: (arg: Error) => void) {
-    while (!this.closed) {
-      this.task.cancel = () => {
-        this.closed = true;
+    while (!this.#closed) {
+      this.#task.cancel = () => {
+        this.#closed = true;
         reject(new Error('cancelled'));
       };
-      for await (const sentence of this.sentenceStream) {
-        const audio = await this.tts.synthesize(sentence.text).then((data) => data.next());
+      for await (const sentence of this.#sentenceStream) {
+        const audio = await this.#tts.synthesize(sentence.text).then((data) => data.next());
         if (!audio.done) {
-          this.eventQueue.push(new SynthesisEvent(SynthesisEventType.STARTED));
-          this.eventQueue.push(new SynthesisEvent(SynthesisEventType.AUDIO, audio.value));
-          this.eventQueue.push(new SynthesisEvent(SynthesisEventType.FINISHED));
+          this.#eventQueue.push(new SynthesisEvent(SynthesisEventType.STARTED));
+          this.#eventQueue.push(new SynthesisEvent(SynthesisEventType.AUDIO, audio.value));
+          this.#eventQueue.push(new SynthesisEvent(SynthesisEventType.FINISHED));
         }
       }
     }
   }
 
   pushText(token: string) {
-    this.sentenceStream.pushText(token);
+    this.#sentenceStream.pushText(token);
   }
 
   async flush() {
-    await this.sentenceStream.flush();
+    await this.#sentenceStream.flush();
   }
 
   next(): IteratorResult<SynthesisEvent> {
-    const event = this.eventQueue.shift();
+    const event = this.#eventQueue.shift();
     if (event) {
       return { done: false, value: event };
     } else {
@@ -63,30 +64,30 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   }
 
   async close(): Promise<void> {
-    this.task.cancel();
+    this.#task.cancel();
     try {
-      await this.task.run;
+      await this.#task.run;
     } finally {
-      this.eventQueue.push(undefined);
+      this.#eventQueue.push(undefined);
     }
   }
 }
 
 export class StreamAdapter extends TTS {
-  tts: TTS;
-  tokenizer: SentenceTokenizer;
+  #tts: TTS;
+  #tokenizer: SentenceTokenizer;
 
   constructor(tts: TTS, tokenizer: SentenceTokenizer) {
     super(true);
-    this.tts = tts;
-    this.tokenizer = tokenizer;
+    this.#tts = tts;
+    this.#tokenizer = tokenizer;
   }
 
   synthesize(text: string): Promise<ChunkedStream> {
-    return this.tts.synthesize(text);
+    return this.#tts.synthesize(text);
   }
 
   stream() {
-    return new StreamAdapterWrapper(this.tts, this.tokenizer.stream(undefined));
+    return new StreamAdapterWrapper(this.#tts, this.#tokenizer.stream(undefined));
   }
 }

package/src/tts/tts.ts

@@ -10,8 +10,18 @@ export interface SynthesizedAudio {
 }
 
 export enum SynthesisEventType {
+  /**
+   * Indicate the start of synthesis.
+   * Retriggered after FINISHED.
+   */
   STARTED = 0,
+  /**
+   * Indicate that audio data is available.
+   */
   AUDIO = 1,
+  /**
+   * Indicate the end of synthesis. Does not necessarily mean stream is done.
+   */
   FINISHED = 2,
 }
 

package/src/utils.ts

@@ -1,10 +1,26 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { AudioFrame } from '@livekit/rtc-node';
+import type {
+  LocalParticipant,
+  RemoteParticipant,
+  Room,
+  TrackPublication,
+} from '@livekit/rtc-node';
+import { AudioFrame, TrackSource } from '@livekit/rtc-node';
+import { EventEmitter, once } from 'events';
 
+/** Union of a single and a list of {@link AudioFrame}s */
 export type AudioBuffer = AudioFrame[] | AudioFrame;
 
+/**
+ * Merge one or more {@link AudioFrame}s into a single one.
+ *
+ * @param buffer Either an {@link AudioFrame} or a list thereof
+ * @throws
+ * {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
+ * | TypeError} if sample rate or channel count are mismatched
+ */
 export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
   if (Array.isArray(buffer)) {
     buffer = buffer as AudioFrame[];
@@ -15,7 +31,7 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
     const sampleRate = buffer[0].sampleRate;
     const channels = buffer[0].channels;
     let samplesPerChannel = 0;
-    let data = new Uint16Array();
+    let data = new Int16Array();
 
     for (const frame of buffer) {
       if (frame.sampleRate !== sampleRate) {
@@ -26,7 +42,7 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
         throw new TypeError('channel count mismatch');
       }
 
-      data = new Uint16Array([...data, ...frame.data]);
+      data = new Int16Array([...data, ...frame.data]);
       samplesPerChannel += frame.samplesPerChannel;
     }
 
@@ -35,3 +51,241 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
 
   return buffer;
 };
+
+export const findMicroTrackId = (room: Room, identity: string): string => {
+  let p: RemoteParticipant | LocalParticipant | undefined = room.remoteParticipants.get(identity);
+
+  if (identity === room.localParticipant?.identity) {
+    p = room.localParticipant;
+  }
+
+  if (!p) {
+    throw new Error(`participant ${identity} not found`);
+  }
+
+  // find first micro track
+  let trackId: string | undefined;
+  p.trackPublications.forEach((track: TrackPublication) => {
+    if (track.source === TrackSource.SOURCE_MICROPHONE) {
+      trackId = track.sid;
+      return;
+    }
+  });
+
+  if (!trackId) {
+    throw new Error(`participant ${identity} does not have a microphone track`);
+  }
+
+  return trackId;
+};
+
+/** @internal */
+export class Mutex {
+  #locking: Promise<void>;
+  #locks: number;
+  #limit: number;
+
+  constructor(limit = 1) {
+    this.#locking = Promise.resolve();
+    this.#locks = 0;
+    this.#limit = limit;
+  }
+
+  isLocked(): boolean {
+    return this.#locks >= this.#limit;
+  }
+
+  async lock(): Promise<() => void> {
+    this.#locks += 1;
+
+    let unlockNext: () => void;
+
+    const willLock = new Promise<void>(
+      (resolve) =>
+        (unlockNext = () => {
+          this.#locks -= 1;
+          resolve();
+        }),
+    );
+
+    const willUnlock = this.#locking.then(() => unlockNext);
+    this.#locking = this.#locking.then(() => willLock);
+    return willUnlock;
+  }
+}
+
+/** @internal */
+export class Queue<T> {
+  /** @internal */
+  items: T[] = [];
+  #limit?: number;
+  #events = new EventEmitter();
+
+  constructor(limit?: number) {
+    this.#limit = limit;
+  }
+
+  async get(): Promise<T> {
+    if (this.items.length === 0) {
+      await once(this.#events, 'put');
+    }
+    const item = this.items.shift()!;
+    this.#events.emit('get');
+    return item;
+  }
+
+  async put(item: T) {
+    if (this.#limit && this.items.length >= this.#limit) {
+      await once(this.#events, 'get');
+    }
+    this.items.push(item);
+    this.#events.emit('put');
+  }
+}
+
+/** @internal */
+export class Future {
+  #await = new Promise<void>((resolve, reject: (_: Error) => void) => {
+    this.resolve = resolve;
+    this.reject = reject;
+  });
+  #done: boolean = false;
+
+  get await() {
+    return this.#await;
+  }
+
+  get done() {
+    return this.#done;
+  }
+
+  resolve() {
+    this.#done = true;
+  }
+
+  reject(_: Error) {
+    this.#done = true;
+    _;
+  }
+}
+
+/** @internal */
+export class CancellablePromise<T> {
+  #promise: Promise<T>;
+  #cancelFn: () => void;
+  #isCancelled: boolean = false;
+  #error: Error | null = null;
+
+  constructor(
+    executor: (
+      resolve: (value: T | PromiseLike<T>) => void,
+      reject: (reason?: any) => void,
+      onCancel: (cancelFn: () => void) => void,
+    ) => void,
+  ) {
+    let cancel: () => void;
+
+    this.#promise = new Promise<T>((resolve, reject) => {
+      executor(
+        resolve,
+        (reason) => {
+          this.#error = reason instanceof Error ? reason : new Error(String(reason));
+          reject(reason);
+        },
+        (cancelFn) => {
+          cancel = () => {
+            this.#isCancelled = true;
+            cancelFn();
+          };
+        },
+      );
+    });
+
+    this.#cancelFn = cancel!;
+  }
+
+  get isCancelled(): boolean {
+    return this.#isCancelled;
+  }
+
+  get error(): Error | null {
+    return this.#error;
+  }
+
+  then<TResult1 = T, TResult2 = never>(
+    onfulfilled?: ((value: T) => TResult1 | Promise<TResult1>) | null,
+    onrejected?: ((reason: any) => TResult2 | Promise<TResult2>) | null,
+  ): Promise<TResult1 | TResult2> {
+    return this.#promise.then(onfulfilled, onrejected);
+  }
+
+  catch<TResult = never>(
+    onrejected?: ((reason: any) => TResult | Promise<TResult>) | null,
+  ): Promise<T | TResult> {
+    return this.#promise.catch(onrejected);
+  }
+
+  finally(onfinally?: (() => void) | null): Promise<T> {
+    return this.#promise.finally(onfinally);
+  }
+
+  cancel(): void {
+    this.#cancelFn();
+  }
+
+  static from<T>(promise: Promise<T>): CancellablePromise<T> {
+    return new CancellablePromise<T>((resolve, reject) => {
+      promise.then(resolve).catch(reject);
+    });
+  }
+}
+
+/** @internal */
+export async function gracefullyCancel<T>(promise: CancellablePromise<T>): Promise<void> {
+  if (!promise.isCancelled) {
+    promise.cancel();
+  }
+  try {
+    await promise;
+  } catch (error) {
+    // Ignore the error, as it's expected due to cancellation
+  }
+}
+
+/** @internal */
+export class AsyncIterableQueue<T> implements AsyncIterable<T> {
+  private queue: Queue<T | typeof AsyncIterableQueue.QUEUE_END_MARKER>;
+  private closed = false;
+  private static readonly QUEUE_END_MARKER = Symbol('QUEUE_END_MARKER');
+
+  constructor() {
+    this.queue = new Queue<T | typeof AsyncIterableQueue.QUEUE_END_MARKER>();
+  }
+
+  put(item: T): void {
+    if (this.closed) {
+      throw new Error('Queue is closed');
+    }
+    this.queue.put(item);
+  }
+
+  close(): void {
+    this.closed = true;
+    this.queue.put(AsyncIterableQueue.QUEUE_END_MARKER);
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    return {
+      next: async (): Promise<IteratorResult<T>> => {
+        if (this.closed && this.queue.items.length === 0) {
+          return { value: undefined, done: true };
+        }
+        const item = await this.queue.get();
+        if (item === AsyncIterableQueue.QUEUE_END_MARKER && this.closed) {
+          return { value: undefined, done: true };
+        }
+        return { value: item as T, done: false };
+      },
+    };
+  }
+}

package/src/vad.ts

@@ -11,12 +11,23 @@ export enum VADEventType {
 
 export interface VADEvent {
   type: VADEventType;
+  /**
+   * Index of the samples of the event (when the event was fired)
+   */
   samplesIndex: number;
+  /**
+   * Duration of speech, in seconds
+   */
   duration: number;
   speech: AudioFrame[];
 }
 
 export abstract class VAD {
+  /**
+   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
+   *
+   * @param options
+   */
   abstract stream({
     minSpeakingDuration,
     minSilenceDuration,
@@ -24,10 +35,28 @@ export abstract class VAD {
     sampleRate,
     maxBufferedSpeech,
   }: {
+    /**
+     * Minimum duration of speech required to trigger a {@link VADEventType.START_OF_SPEECH} event
+     */
     minSpeakingDuration: number;
+    /**
+     * Milliseconds to wait before separating speech chunk.
+     * Not always precise, generally rounded to the nearest 40ms depending on VAD implementation
+     */
     minSilenceDuration: number;
+    /**
+     * Number of frames to pad the start and end of speech with
+     */
     paddingDuration: number;
+    /**
+     * Sample rate of inference/processing
+     */
     sampleRate: number;
+    /**
+     * Number of seconds the buffer may keep until {@link VADEventType.END_OF_SPEECH} is triggered.
+     * It is recommended to set this to a positive value, as zero may OOM if the user doesn't stop
+     * speaking.
+     */
     maxBufferedSpeech: number;
   }): VADStream;
 }