@livekit/agents 0.1.0 → 0.2.0

package/src/stt/stream_adapter.ts

@@ -7,24 +7,24 @@ import { VADEventType, type VADStream } from '../vad.js';
 import { STT, SpeechEvent, SpeechEventType, SpeechStream } from './stt.js';
 
 export class StreamAdapterWrapper extends SpeechStream {
-  closed: boolean;
-  stt: STT;
-  vadStream: VADStream;
-  eventQueue: (SpeechEvent | undefined)[];
-  language?: string;
-  task: {
+  #closed: boolean;
+  #stt: STT;
+  #vadStream: VADStream;
+  #eventQueue: (SpeechEvent | undefined)[];
+  #language?: string;
+  #task: {
     run: Promise<void>;
     cancel: () => void;
   };
 
   constructor(stt: STT, vadStream: VADStream, language: string | undefined = undefined) {
     super();
-    this.closed = false;
-    this.stt = stt;
-    this.vadStream = vadStream;
-    this.eventQueue = [];
-    this.language = language;
-    this.task = {
+    this.#closed = false;
+    this.#stt = stt;
+    this.#vadStream = vadStream;
+    this.#eventQueue = [];
+    this.#language = language;
+    this.#task = {
       run: new Promise((_, reject) => {
         this.run(reject);
       }),
@@ -33,46 +33,46 @@ export class StreamAdapterWrapper extends SpeechStream {
   }
 
   async run(reject: (arg: Error) => void) {
-    this.task.cancel = () => {
-      this.closed = true;
+    this.#task.cancel = () => {
+      this.#closed = true;
       reject(new Error('cancelled'));
     };
 
-    for (const event of this.vadStream) {
+    for (const event of this.#vadStream) {
       if (event.type == VADEventType.START_OF_SPEECH) {
         const startEvent = new SpeechEvent(SpeechEventType.START_OF_SPEECH);
-        this.eventQueue.push(startEvent);
+        this.#eventQueue.push(startEvent);
       } else if (event.type == VADEventType.END_OF_SPEECH) {
         const mergedFrames = mergeFrames(event.speech);
-        const endEvent = await this.stt.recognize(mergedFrames, this.language);
-        this.eventQueue.push(endEvent);
+        const endEvent = await this.#stt.recognize(mergedFrames, this.#language);
+        this.#eventQueue.push(endEvent);
       }
     }
 
-    this.eventQueue.push(undefined);
+    this.#eventQueue.push(undefined);
   }
 
   pushFrame(frame: AudioFrame) {
-    if (this.closed) {
+    if (this.#closed) {
       throw new TypeError('cannot push frame to closed stream');
     }
 
-    this.vadStream.pushFrame(frame);
+    this.#vadStream.pushFrame(frame);
   }
 
   async close(wait: boolean = true): Promise<void> {
-    this.closed = true;
+    this.#closed = true;
 
     if (!wait) {
-      this.task.cancel();
+      this.#task.cancel();
     }
 
-    await this.vadStream.close(wait);
-    await this.task.run;
+    await this.#vadStream.close(wait);
+    await this.#task.run;
   }
 
   next(): IteratorResult<SpeechEvent> {
-    const item = this.eventQueue.shift();
+    const item = this.#eventQueue.shift();
     if (item) {
       return { done: false, value: item };
     } else {
@@ -82,23 +82,23 @@ export class StreamAdapterWrapper extends SpeechStream {
 }
 
 export class StreamAdapter extends STT {
-  stt: STT;
-  vadStream: VADStream;
+  #stt: STT;
+  #vadStream: VADStream;
 
   constructor(stt: STT, vadStream: VADStream) {
     super(true);
-    this.stt = stt;
-    this.vadStream = vadStream;
+    this.#stt = stt;
+    this.#vadStream = vadStream;
   }
 
   async recognize(
     buffer: AudioBuffer,
     language: string | undefined = undefined,
   ): Promise<SpeechEvent> {
-    return await this.stt.recognize(buffer, language);
+    return await this.#stt.recognize(buffer, language);
   }
 
   stream(language: string | undefined = undefined) {
-    return new StreamAdapterWrapper(this.stt, this.vadStream, language);
+    return new StreamAdapterWrapper(this.#stt, this.#vadStream, language);
   }
 }

package/src/stt/stt.ts

@@ -5,9 +5,25 @@ import type { AudioFrame } from '@livekit/rtc-node';
 import type { AudioBuffer } from '../utils.js';
 
 export enum SpeechEventType {
+  /**
+   * Indicate the start of speech.
+   * If the STT doesn't support this event, this will be emitted at the same time
+   * as the first INTERMIN_TRANSCRIPT.
+   */
   START_OF_SPEECH = 0,
+  /**
+   * Interim transcript, useful for real-time transcription.
+   */
   INTERIM_TRANSCRIPT = 1,
+  /**
+   * Final transcript, emitted when the STT is confident enough that a certain
+   * portion of the speech will not change.
+   */
   FINAL_TRANSCRIPT = 2,
+  /**
+   * Indicate the end of speech, emitted when the user stops speaking.
+   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.
+   */
   END_OF_SPEECH = 3,
 }
 
@@ -30,8 +46,19 @@ export class SpeechEvent {
 }
 
 export abstract class SpeechStream implements IterableIterator<SpeechEvent> {
+  /**
+   * Push a frame to be recognised.
+   * It is recommended to push frames as soon as they are available.
+   */
   abstract pushFrame(token: AudioFrame): void;
 
+  /**
+   * Close the stream.
+   *
+   * @param wait
+   *   Whether to wait for the STT to finish processing the remaining
+   *   frames before closing
+   */
   abstract close(wait: boolean): Promise<void>;
 
   abstract next(): IteratorResult<SpeechEvent>;

package/src/tts/stream_adapter.ts

@@ -2,25 +2,26 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { SentenceStream, SentenceTokenizer } from '../tokenize.js';
-import { ChunkedStream, SynthesisEvent, SynthesisEventType, SynthesizeStream, TTS } from './tts.js';
+import type { ChunkedStream } from './tts.js';
+import { SynthesisEvent, SynthesisEventType, SynthesizeStream, TTS } from './tts.js';
 
 export class StreamAdapterWrapper extends SynthesizeStream {
-  closed: boolean;
-  tts: TTS;
-  sentenceStream: SentenceStream;
-  eventQueue: (SynthesisEvent | undefined)[];
-  task: {
+  #closed: boolean;
+  #tts: TTS;
+  #sentenceStream: SentenceStream;
+  #eventQueue: (SynthesisEvent | undefined)[];
+  #task: {
     run: Promise<void>;
     cancel: () => void;
   };
 
   constructor(tts: TTS, sentenceStream: SentenceStream) {
     super();
-    this.closed = false;
-    this.tts = tts;
-    this.sentenceStream = sentenceStream;
-    this.eventQueue = [];
-    this.task = {
+    this.#closed = false;
+    this.#tts = tts;
+    this.#sentenceStream = sentenceStream;
+    this.#eventQueue = [];
+    this.#task = {
       run: new Promise((_, reject) => {
         this.run(reject);
       }),
@@ -29,32 +30,32 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   }
 
   async run(reject: (arg: Error) => void) {
-    while (!this.closed) {
-      this.task.cancel = () => {
-        this.closed = true;
+    while (!this.#closed) {
+      this.#task.cancel = () => {
+        this.#closed = true;
         reject(new Error('cancelled'));
       };
-      for await (const sentence of this.sentenceStream) {
-        const audio = await this.tts.synthesize(sentence.text).then((data) => data.next());
+      for await (const sentence of this.#sentenceStream) {
+        const audio = await this.#tts.synthesize(sentence.text).then((data) => data.next());
         if (!audio.done) {
-          this.eventQueue.push(new SynthesisEvent(SynthesisEventType.STARTED));
-          this.eventQueue.push(new SynthesisEvent(SynthesisEventType.AUDIO, audio.value));
-          this.eventQueue.push(new SynthesisEvent(SynthesisEventType.FINISHED));
+          this.#eventQueue.push(new SynthesisEvent(SynthesisEventType.STARTED));
+          this.#eventQueue.push(new SynthesisEvent(SynthesisEventType.AUDIO, audio.value));
+          this.#eventQueue.push(new SynthesisEvent(SynthesisEventType.FINISHED));
         }
       }
     }
   }
 
   pushText(token: string) {
-    this.sentenceStream.pushText(token);
+    this.#sentenceStream.pushText(token);
   }
 
   async flush() {
-    await this.sentenceStream.flush();
+    await this.#sentenceStream.flush();
   }
 
   next(): IteratorResult<SynthesisEvent> {
-    const event = this.eventQueue.shift();
+    const event = this.#eventQueue.shift();
     if (event) {
       return { done: false, value: event };
     } else {
@@ -63,30 +64,30 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   }
 
   async close(): Promise<void> {
-    this.task.cancel();
+    this.#task.cancel();
     try {
-      await this.task.run;
+      await this.#task.run;
     } finally {
-      this.eventQueue.push(undefined);
+      this.#eventQueue.push(undefined);
     }
   }
 }
 
 export class StreamAdapter extends TTS {
-  tts: TTS;
-  tokenizer: SentenceTokenizer;
+  #tts: TTS;
+  #tokenizer: SentenceTokenizer;
 
   constructor(tts: TTS, tokenizer: SentenceTokenizer) {
     super(true);
-    this.tts = tts;
-    this.tokenizer = tokenizer;
+    this.#tts = tts;
+    this.#tokenizer = tokenizer;
   }
 
   synthesize(text: string): Promise<ChunkedStream> {
-    return this.tts.synthesize(text);
+    return this.#tts.synthesize(text);
   }
 
   stream() {
-    return new StreamAdapterWrapper(this.tts, this.tokenizer.stream(undefined));
+    return new StreamAdapterWrapper(this.#tts, this.#tokenizer.stream(undefined));
   }
 }

package/src/tts/tts.ts

@@ -10,8 +10,18 @@ export interface SynthesizedAudio {
 }
 
 export enum SynthesisEventType {
+  /**
+   * Indicate the start of synthesis.
+   * Retriggered after FINISHED.
+   */
   STARTED = 0,
+  /**
+   * Indicate that audio data is available.
+   */
   AUDIO = 1,
+  /**
+   * Indicate the end of synthesis. Does not necessarily mean stream is done.
+   */
   FINISHED = 2,
 }
 

package/src/utils.ts

@@ -1,10 +1,26 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { AudioFrame } from '@livekit/rtc-node';
+import type {
+  LocalParticipant,
+  RemoteParticipant,
+  Room,
+  TrackPublication,
+} from '@livekit/rtc-node';
+import { AudioFrame, TrackSource } from '@livekit/rtc-node';
+import { EventEmitter, once } from 'events';
 
+/** Union of a single and a list of {@link AudioFrame}s */
 export type AudioBuffer = AudioFrame[] | AudioFrame;
 
+/**
+ * Merge one or more {@link AudioFrame}s into a single one.
+ *
+ * @param buffer Either an {@link AudioFrame} or a list thereof
+ * @throws
+ * {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
+ * | TypeError} if sample rate or channel count are mismatched
+ */
 export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
   if (Array.isArray(buffer)) {
     buffer = buffer as AudioFrame[];
@@ -15,7 +31,7 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
     const sampleRate = buffer[0].sampleRate;
     const channels = buffer[0].channels;
     let samplesPerChannel = 0;
-    let data = new Uint16Array();
+    let data = new Int16Array();
 
     for (const frame of buffer) {
       if (frame.sampleRate !== sampleRate) {
@@ -26,7 +42,7 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
         throw new TypeError('channel count mismatch');
       }
 
-      data = new Uint16Array([...data, ...frame.data]);
+      data = new Int16Array([...data, ...frame.data]);
       samplesPerChannel += frame.samplesPerChannel;
     }
 
@@ -35,3 +51,109 @@ export const mergeFrames = (buffer: AudioBuffer): AudioFrame => {
 
   return buffer;
 };
+
+export const findMicroTrackId = (room: Room, identity: string): string => {
+  let p: RemoteParticipant | LocalParticipant | undefined = room.remoteParticipants.get(identity);
+
+  if (identity === room.localParticipant?.identity) {
+    p = room.localParticipant;
+  }
+
+  if (!p) {
+    throw new Error(`participant ${identity} not found`);
+  }
+
+  // find first micro track
+  let trackId: string | undefined;
+  p.trackPublications.forEach((track: TrackPublication) => {
+    if (track.source === TrackSource.SOURCE_MICROPHONE) {
+      trackId = track.sid;
+      return;
+    }
+  });
+
+  if (!trackId) {
+    throw new Error(`participant ${identity} does not have a microphone track`);
+  }
+
+  return trackId;
+};
+
+/** @internal */
+export class Mutex {
+  #locking: Promise<void>;
+  #locks: number;
+  #limit: number;
+
+  constructor(limit = 1) {
+    this.#locking = Promise.resolve();
+    this.#locks = 0;
+    this.#limit = limit;
+  }
+
+  isLocked(): boolean {
+    return this.#locks >= this.#limit;
+  }
+
+  async lock(): Promise<() => void> {
+    this.#locks += 1;
+
+    let unlockNext: () => void;
+
+    const willLock = new Promise<void>(
+      (resolve) =>
+        (unlockNext = () => {
+          this.#locks -= 1;
+          resolve();
+        }),
+    );
+
+    const willUnlock = this.#locking.then(() => unlockNext);
+    this.#locking = this.#locking.then(() => willLock);
+    return willUnlock;
+  }
+}
+
+/** @internal */
+export class Queue<T> {
+  #items: T[] = [];
+  #limit?: number;
+  #events = new EventEmitter();
+
+  constructor(limit?: number) {
+    this.#limit = limit;
+  }
+
+  async get(): Promise<T> {
+    if (this.#items.length === 0) {
+      await once(this.#events, 'put');
+    }
+    const item = this.#items.shift()!;
+    this.#events.emit('get');
+    return item;
+  }
+
+  async put(item: T) {
+    if (this.#limit && this.#items.length >= this.#limit) {
+      await once(this.#events, 'get');
+    }
+    this.#items.push(item);
+    this.#events.emit('put');
+  }
+}
+
+/** @internal */
+export class Future {
+  #await = new Promise<void>((resolve, reject: (_: Error) => void) => {
+    this.resolve = resolve;
+    this.reject = reject;
+  });
+
+  get await() {
+    return this.#await;
+  }
+  resolve() {}
+  reject(_: Error) {
+    _;
+  }
+}

package/src/vad.ts

@@ -11,12 +11,23 @@ export enum VADEventType {
 
 export interface VADEvent {
   type: VADEventType;
+  /**
+   * Index of the samples of the event (when the event was fired)
+   */
   samplesIndex: number;
+  /**
+   * Duration of speech, in seconds
+   */
   duration: number;
   speech: AudioFrame[];
 }
 
 export abstract class VAD {
+  /**
+   * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
+   *
+   * @param options
+   */
   abstract stream({
     minSpeakingDuration,
     minSilenceDuration,
@@ -24,10 +35,28 @@ export abstract class VAD {
     sampleRate,
     maxBufferedSpeech,
   }: {
+    /**
+     * Minimum duration of speech required to trigger a {@link VADEventType.START_OF_SPEECH} event
+     */
     minSpeakingDuration: number;
+    /**
+     * Milliseconds to wait before separating speech chunk.
+     * Not always precise, generally rounded to the nearest 40ms depending on VAD implementation
+     */
     minSilenceDuration: number;
+    /**
+     * Number of frames to pad the start and end of speech with
+     */
     paddingDuration: number;
+    /**
+     * Sample rate of inference/processing
+     */
     sampleRate: number;
+    /**
+     * Number of seconds the buffer may keep until {@link VADEventType.END_OF_SPEECH} is triggered.
+     * It is recommended to set this to a positive value, as zero may OOM if the user doesn't stop
+     * speaking.
+     */
     maxBufferedSpeech: number;
   }): VADStream;
 }