@livekit/agents 0.4.6 → 0.5.1

package/dist/pipeline/human_input.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/pipeline/human_input.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { log } from '../log.js';\nimport type { STT, SpeechEvent } from '../stt/stt.js';\nimport { SpeechEventType } from '../stt/stt.js';\nimport { CancellablePromise, gracefullyCancel } from '../utils.js';\nimport type { VAD, VADEvent } from '../vad.js';\nimport { VADEventType } from '../vad.js';\n\nexport enum HumanInputEvent {\n  START_OF_SPEECH,\n  VAD_INFERENCE_DONE,\n  END_OF_SPEECH,\n  FINAL_TRANSCRIPT,\n  INTERIM_TRANSCRIPT,\n}\n\nexport type HumanInputCallbacks = {\n  [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;\n  [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;\n  [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;\n};\n\nexport class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInputCallbacks>) {\n  #closed = false;\n  #room: Room;\n  #vad: VAD;\n  #stt: STT;\n  #participant: RemoteParticipant;\n  #subscribedTrack?: RemoteAudioTrack;\n  #recognizeTask?: CancellablePromise<void>;\n  #speaking = false;\n  #speechProbability = 0;\n  #logger = log();\n\n  constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant) {\n    super();\n    this.#room = room;\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#participant = participant;\n\n    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));\n    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));\n    this.#subscribeToMicrophone();\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.#participant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.#participant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n\n    const track = microphonePublication.track;\n    if (track && track !== this.#subscribedTrack) {\n      this.#subscribedTrack = track;\n      if (this.#recognizeTask) {\n        this.#recognizeTask.cancel();\n      }\n\n      const audioStream = new AudioStream(track, 16000);\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        const sttStream = this.#stt.stream();\n        const vadStream = this.#vad.stream();\n\n        const audioStreamCo = async () => {\n          for await (const ev of audioStream) {\n            if (cancelled) return;\n            sttStream.pushFrame(ev);\n            vadStream.pushFrame(ev);\n          }\n        };\n\n        const vadStreamCo = async () => {\n          for await (const ev of vadStream) {\n            if (cancelled) return;\n            switch (ev.type) {\n              case VADEventType.START_OF_SPEECH:\n                this.#speaking = true;\n                this.emit(HumanInputEvent.START_OF_SPEECH, ev);\n                break;\n              case VADEventType.INFERENCE_DONE:\n                this.#speechProbability = ev.probability;\n                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);\n                break;\n              case VADEventType.END_OF_SPEECH:\n                this.#speaking = false;\n                this.emit(HumanInputEvent.END_OF_SPEECH, ev);\n                break;\n            }\n          }\n        };\n\n        const sttStreamCo = async () => {\n          for await (const ev of sttStream) {\n            if (cancelled) return;\n            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {\n              this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);\n            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {\n              this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);\n            }\n          }\n        };\n\n        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);\n        sttStream.close();\n        vadStream.close();\n        resolve();\n      });\n    }\n  }\n\n  get speaking(): boolean {\n    return this.#speaking;\n  }\n\n  get speakingProbability(): number {\n    return this.#speechProbability;\n  }\n\n  async close() {\n    if (this.#closed) {\n      throw new Error('HumanInput already closed');\n    }\n    this.#closed = true;\n    this.#room.removeAllListeners();\n    this.#speaking = false;\n    if (this.#recognizeTask) {\n      await gracefullyCancel(this.#recognizeTask);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASA,sBAAoD;AAEpD,yBAA6B;AAC7B,iBAAoB;AAEpB,iBAAgC;AAChC,mBAAqD;AAErD,iBAA6B;AAEtB,IAAK,kBAAL,kBAAKA,qBAAL;AACL,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AALU,SAAAA;AAAA,GAAA;AAgBL,MAAM,mBAAoB,gCAA6D;AAAA,EAC5F,UAAU;AAAA,EACV;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,qBAAqB;AAAA,EACrB,cAAU,gBAAI;AAAA,EAEd,YAAY,MAAY,KAAU,KAAU,aAAgC;AAC1E,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,MAAM,GAAG,0BAAU,gBAAgB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC9E,SAAK,MAAM,GAAG,0BAAU,iBAAiB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC/E,SAAK,uBAAuB;AAAA,EAC9B;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,aAAa,kBAAkB,OAAO,GAAG;AACtE,UAAI,YAAY,WAAW,4BAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAEA,UAAM,QAAQ,sBAAsB;AACpC,QAAI,SAAS,UAAU,KAAK,kBAAkB;AAC5C,WAAK,mBAAmB;AACxB,UAAI,KAAK,gBAAgB;AACvB,aAAK,eAAe,OAAO;AAAA,MAC7B;AAEA,YAAM,cAAc,IAAI,4BAAY,OAAO,IAAK;AAGhD,WAAK,iBAAiB,IAAI,gCAAmB,OAAO,SAAS,GAAG,aAAa;AAC3E,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,cAAM,YAAY,KAAK,KAAK,OAAO;AACnC,cAAM,YAAY,KAAK,KAAK,OAAO;AAEnC,cAAM,gBAAgB,YAAY;AAChC,2BAAiB,MAAM,aAAa;AAClC,gBAAI,UAAW;AACf,sBAAU,UAAU,EAAE;AACtB,sBAAU,UAAU,EAAE;AAAA,UACxB;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,oBAAQ,GAAG,MAAM;AAAA,cACf,KAAK,wBAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,yBAAiC,EAAE;AAC7C;AAAA,cACF,KAAK,wBAAa;AAChB,qBAAK,qBAAqB,GAAG;AAC7B,qBAAK,KAAK,4BAAoC,EAAE;AAChD;AAAA,cACF,KAAK,wBAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,uBAA+B,EAAE;AAC3C;AAAA,YACJ;AAAA,UACF;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,gBAAI,GAAG,SAAS,2BAAgB,kBAAkB;AAChD,mBAAK,KAAK,0BAAkC,EAAE;AAAA,YAChD,WAAW,GAAG,QAAQ,2BAAgB,oBAAoB;AACxD,mBAAK,KAAK,4BAAoC,EAAE;AAAA,YAClD;AAAA,UACF;AAAA,QACF;AAEA,cAAM,QAAQ,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC,CAAC;AACjE,kBAAU,MAAM;AAChB,kBAAU,MAAM;AAChB,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AACA,SAAK,UAAU;AACf,SAAK,MAAM,mBAAmB;AAC9B,SAAK,YAAY;AACjB,QAAI,KAAK,gBAAgB;AACvB,gBAAM,+BAAiB,KAAK,cAAc;AAAA,IAC5C;AAAA,EACF;AACF;","names":["HumanInputEvent"]}

package/dist/pipeline/human_input.js

@@ -1,134 +1,133 @@
-import { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';
-import { EventEmitter } from 'node:events';
-import { log } from '../log.js';
-import { SpeechEventType } from '../stt/stt.js';
-import { CancellablePromise, gracefullyCancel } from '../utils.js';
-import { VADEventType } from '../vad.js';
-export var HumanInputEvent;
-(function (HumanInputEvent) {
-    HumanInputEvent[HumanInputEvent["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
-    HumanInputEvent[HumanInputEvent["VAD_INFERENCE_DONE"] = 1] = "VAD_INFERENCE_DONE";
-    HumanInputEvent[HumanInputEvent["END_OF_SPEECH"] = 2] = "END_OF_SPEECH";
-    HumanInputEvent[HumanInputEvent["FINAL_TRANSCRIPT"] = 3] = "FINAL_TRANSCRIPT";
-    HumanInputEvent[HumanInputEvent["INTERIM_TRANSCRIPT"] = 4] = "INTERIM_TRANSCRIPT";
-})(HumanInputEvent || (HumanInputEvent = {}));
-export class HumanInput extends EventEmitter {
-    #closed = false;
-    #room;
-    #vad;
-    #stt;
-    #participant;
-    #subscribedTrack;
-    #recognizeTask;
-    #speaking = false;
-    #speechProbability = 0;
-    #logger = log();
-    constructor(room, vad, stt, participant) {
-        super();
-        this.#room = room;
-        this.#vad = vad;
-        this.#stt = stt;
-        this.#participant = participant;
-        this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));
-        this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));
-        this.#subscribeToMicrophone();
+import { AudioStream, RoomEvent, TrackSource } from "@livekit/rtc-node";
+import { EventEmitter } from "node:events";
+import { log } from "../log.js";
+import { SpeechEventType } from "../stt/stt.js";
+import { CancellablePromise, gracefullyCancel } from "../utils.js";
+import { VADEventType } from "../vad.js";
+var HumanInputEvent = /* @__PURE__ */ ((HumanInputEvent2) => {
+  HumanInputEvent2[HumanInputEvent2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
+  HumanInputEvent2[HumanInputEvent2["VAD_INFERENCE_DONE"] = 1] = "VAD_INFERENCE_DONE";
+  HumanInputEvent2[HumanInputEvent2["END_OF_SPEECH"] = 2] = "END_OF_SPEECH";
+  HumanInputEvent2[HumanInputEvent2["FINAL_TRANSCRIPT"] = 3] = "FINAL_TRANSCRIPT";
+  HumanInputEvent2[HumanInputEvent2["INTERIM_TRANSCRIPT"] = 4] = "INTERIM_TRANSCRIPT";
+  return HumanInputEvent2;
+})(HumanInputEvent || {});
+class HumanInput extends EventEmitter {
+  #closed = false;
+  #room;
+  #vad;
+  #stt;
+  #participant;
+  #subscribedTrack;
+  #recognizeTask;
+  #speaking = false;
+  #speechProbability = 0;
+  #logger = log();
+  constructor(room, vad, stt, participant) {
+    super();
+    this.#room = room;
+    this.#vad = vad;
+    this.#stt = stt;
+    this.#participant = participant;
+    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));
+    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));
+    this.#subscribeToMicrophone();
+  }
+  #subscribeToMicrophone() {
+    if (!this.#participant) {
+      this.#logger.error("Participant is not set");
+      return;
     }
-    #subscribeToMicrophone() {
-        if (!this.#participant) {
-            this.#logger.error('Participant is not set');
-            return;
-        }
-        let microphonePublication = undefined;
-        for (const publication of this.#participant.trackPublications.values()) {
-            if (publication.source === TrackSource.SOURCE_MICROPHONE) {
-                microphonePublication = publication;
+    let microphonePublication = void 0;
+    for (const publication of this.#participant.trackPublications.values()) {
+      if (publication.source === TrackSource.SOURCE_MICROPHONE) {
+        microphonePublication = publication;
+        break;
+      }
+    }
+    if (!microphonePublication) {
+      return;
+    }
+    if (!microphonePublication.subscribed) {
+      microphonePublication.setSubscribed(true);
+    }
+    const track = microphonePublication.track;
+    if (track && track !== this.#subscribedTrack) {
+      this.#subscribedTrack = track;
+      if (this.#recognizeTask) {
+        this.#recognizeTask.cancel();
+      }
+      const audioStream = new AudioStream(track, 16e3);
+      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {
+        let cancelled = false;
+        onCancel(() => {
+          cancelled = true;
+        });
+        const sttStream = this.#stt.stream();
+        const vadStream = this.#vad.stream();
+        const audioStreamCo = async () => {
+          for await (const ev of audioStream) {
+            if (cancelled) return;
+            sttStream.pushFrame(ev);
+            vadStream.pushFrame(ev);
+          }
+        };
+        const vadStreamCo = async () => {
+          for await (const ev of vadStream) {
+            if (cancelled) return;
+            switch (ev.type) {
+              case VADEventType.START_OF_SPEECH:
+                this.#speaking = true;
+                this.emit(0 /* START_OF_SPEECH */, ev);
+                break;
+              case VADEventType.INFERENCE_DONE:
+                this.#speechProbability = ev.probability;
+                this.emit(1 /* VAD_INFERENCE_DONE */, ev);
+                break;
+              case VADEventType.END_OF_SPEECH:
+                this.#speaking = false;
+                this.emit(2 /* END_OF_SPEECH */, ev);
                 break;
             }
-        }
-        if (!microphonePublication) {
-            return;
-        }
-        if (!microphonePublication.subscribed) {
-            microphonePublication.setSubscribed(true);
-        }
-        const track = microphonePublication.track;
-        if (track && track !== this.#subscribedTrack) {
-            this.#subscribedTrack = track;
-            if (this.#recognizeTask) {
-                this.#recognizeTask.cancel();
+          }
+        };
+        const sttStreamCo = async () => {
+          for await (const ev of sttStream) {
+            if (cancelled) return;
+            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {
+              this.emit(3 /* FINAL_TRANSCRIPT */, ev);
+            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {
+              this.emit(4 /* INTERIM_TRANSCRIPT */, ev);
             }
-            const audioStream = new AudioStream(track, 16000);
-            // eslint-disable-next-line @typescript-eslint/no-unused-vars
-            this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {
-                let cancelled = false;
-                onCancel(() => {
-                    cancelled = true;
-                });
-                const sttStream = this.#stt.stream();
-                const vadStream = this.#vad.stream();
-                const audioStreamCo = async () => {
-                    for await (const ev of audioStream) {
-                        if (cancelled)
-                            return;
-                        sttStream.pushFrame(ev);
-                        vadStream.pushFrame(ev);
-                    }
-                };
-                const vadStreamCo = async () => {
-                    for await (const ev of vadStream) {
-                        if (cancelled)
-                            return;
-                        switch (ev.type) {
-                            case VADEventType.START_OF_SPEECH:
-                                this.#speaking = true;
-                                this.emit(HumanInputEvent.START_OF_SPEECH, ev);
-                                break;
-                            case VADEventType.INFERENCE_DONE:
-                                this.#speechProbability = ev.probability;
-                                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);
-                                break;
-                            case VADEventType.END_OF_SPEECH:
-                                this.#speaking = false;
-                                this.emit(HumanInputEvent.END_OF_SPEECH, ev);
-                                break;
-                        }
-                    }
-                };
-                const sttStreamCo = async () => {
-                    for await (const ev of sttStream) {
-                        if (cancelled)
-                            return;
-                        if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {
-                            this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);
-                        }
-                        else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {
-                            this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);
-                        }
-                    }
-                };
-                await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);
-                sttStream.close();
-                vadStream.close();
-                resolve();
-            });
-        }
-    }
-    get speaking() {
-        return this.#speaking;
+          }
+        };
+        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);
+        sttStream.close();
+        vadStream.close();
+        resolve();
+      });
     }
-    get speakingProbability() {
-        return this.#speechProbability;
+  }
+  get speaking() {
+    return this.#speaking;
+  }
+  get speakingProbability() {
+    return this.#speechProbability;
+  }
+  async close() {
+    if (this.#closed) {
+      throw new Error("HumanInput already closed");
     }
-    async close() {
-        if (this.#closed) {
-            throw new Error('HumanInput already closed');
-        }
-        this.#closed = true;
-        this.#room.removeAllListeners();
-        this.#speaking = false;
-        if (this.#recognizeTask) {
-            await gracefullyCancel(this.#recognizeTask);
-        }
+    this.#closed = true;
+    this.#room.removeAllListeners();
+    this.#speaking = false;
+    if (this.#recognizeTask) {
+      await gracefullyCancel(this.#recognizeTask);
     }
+  }
 }
+export {
+  HumanInput,
+  HumanInputEvent
+};
 //# sourceMappingURL=human_input.js.map

package/dist/pipeline/human_input.js.map

@@ -1 +1 @@
-{"version":3,"file":"human_input.js","sourceRoot":"","sources":["../../src/pipeline/human_input.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAExE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEnE,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,MAAM,CAAN,IAAY,eAMX;AAND,WAAY,eAAe;IACzB,2EAAe,CAAA;IACf,iFAAkB,CAAA;IAClB,uEAAa,CAAA;IACb,6EAAgB,CAAA;IAChB,iFAAkB,CAAA;AACpB,CAAC,EANW,eAAe,KAAf,eAAe,QAM1B;AAUD,MAAM,OAAO,UAAW,SAAS,YAA4D;IAC3F,OAAO,GAAG,KAAK,CAAC;IAChB,KAAK,CAAO;IACZ,IAAI,CAAM;IACV,IAAI,CAAM;IACV,YAAY,CAAoB;IAChC,gBAAgB,CAAoB;IACpC,cAAc,CAA4B;IAC1C,SAAS,GAAG,KAAK,CAAC;IAClB,kBAAkB,GAAG,CAAC,CAAC;IACvB,OAAO,GAAG,GAAG,EAAE,CAAC;IAEhB,YAAY,IAAU,EAAE,GAAQ,EAAE,GAAQ,EAAE,WAA8B;QACxE,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAEhC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAChF,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACjF,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAChC,CAAC;IAED,sBAAsB;QACpB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,IAAI,qBAAqB,GAAuC,SAAS,CAAC;QAC1E,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,CAAC,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YACvE,IAAI,WAAW,CAAC,MAAM,KAAK,WAAW,CAAC,iBAAiB,EAAE,CAAC;gBACzD,qBAAqB,GAAG,WAAW,CAAC;gBACpC,MAAM;YACR,CAAC;QACH,CAAC;QACD,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,CAAC;YACtC,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,KAAK,GAAG,qBAAqB,CAAC,KAAK,CAAC;QAC1C,IAAI,KAAK,IAAI,KAAK,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;YAC/B,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAElD,6DAA6D;YAC7D,IAAI,CAAC,cAAc,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;gBAC1E,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,QAAQ,CAAC,GAAG,EAAE;oBACZ,SAAS,GAAG,IAAI,CAAC;gBACnB,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAErC,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;oBAC/B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;wBACnC,IAAI,SAAS;4BAAE,OAAO;wBACtB,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;wBACxB,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC,CAAC;gBAEF,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;oBAC7B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;wBACjC,IAAI,SAAS;4BAAE,OAAO;wBACtB,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;4BAChB,KAAK,YAAY,CAAC,eAAe;gCAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gCACtB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;gCAC/C,MAAM;4BACR,KAAK,YAAY,CAAC,cAAc;gCAC9B,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,WAAW,CAAC;gCACzC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;gCAClD,MAAM;4BACR,KAAK,YAAY,CAAC,aAAa;gCAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gCACvB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;gCAC7C,MAAM;wBACV,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC;gBAEF,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;oBAC7B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;wBACjC,IAAI,SAAS;4BAAE,OAAO;wBACtB,IAAI,EAAE,CAAC,IAAI,KAAK,eAAe,CAAC,gBAAgB,EAAE,CAAC;4BACjD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;wBAClD,CAAC;6BAAM,IAAI,EAAE,CAAC,IAAI,IAAI,eAAe,CAAC,kBAAkB,EAAE,CAAC;4BACzD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;wBACpD,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC;gBAEF,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;gBACnE,SAAS,CAAC,KAAK,EAAE,CAAC;gBAClB,SAAS,CAAC,KAAK,EAAE,CAAC;gBAClB,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;CACF"}
+{"version":3,"sources":["../../src/pipeline/human_input.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  RemoteAudioTrack,\n  RemoteParticipant,\n  RemoteTrackPublication,\n  Room,\n} from '@livekit/rtc-node';\nimport { AudioStream, RoomEvent, TrackSource } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { log } from '../log.js';\nimport type { STT, SpeechEvent } from '../stt/stt.js';\nimport { SpeechEventType } from '../stt/stt.js';\nimport { CancellablePromise, gracefullyCancel } from '../utils.js';\nimport type { VAD, VADEvent } from '../vad.js';\nimport { VADEventType } from '../vad.js';\n\nexport enum HumanInputEvent {\n  START_OF_SPEECH,\n  VAD_INFERENCE_DONE,\n  END_OF_SPEECH,\n  FINAL_TRANSCRIPT,\n  INTERIM_TRANSCRIPT,\n}\n\nexport type HumanInputCallbacks = {\n  [HumanInputEvent.START_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.VAD_INFERENCE_DONE]: (event: VADEvent) => void;\n  [HumanInputEvent.END_OF_SPEECH]: (event: VADEvent) => void;\n  [HumanInputEvent.FINAL_TRANSCRIPT]: (event: SpeechEvent) => void;\n  [HumanInputEvent.INTERIM_TRANSCRIPT]: (event: SpeechEvent) => void;\n};\n\nexport class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInputCallbacks>) {\n  #closed = false;\n  #room: Room;\n  #vad: VAD;\n  #stt: STT;\n  #participant: RemoteParticipant;\n  #subscribedTrack?: RemoteAudioTrack;\n  #recognizeTask?: CancellablePromise<void>;\n  #speaking = false;\n  #speechProbability = 0;\n  #logger = log();\n\n  constructor(room: Room, vad: VAD, stt: STT, participant: RemoteParticipant) {\n    super();\n    this.#room = room;\n    this.#vad = vad;\n    this.#stt = stt;\n    this.#participant = participant;\n\n    this.#room.on(RoomEvent.TrackPublished, this.#subscribeToMicrophone.bind(this));\n    this.#room.on(RoomEvent.TrackSubscribed, this.#subscribeToMicrophone.bind(this));\n    this.#subscribeToMicrophone();\n  }\n\n  #subscribeToMicrophone(): void {\n    if (!this.#participant) {\n      this.#logger.error('Participant is not set');\n      return;\n    }\n\n    let microphonePublication: RemoteTrackPublication | undefined = undefined;\n    for (const publication of this.#participant.trackPublications.values()) {\n      if (publication.source === TrackSource.SOURCE_MICROPHONE) {\n        microphonePublication = publication;\n        break;\n      }\n    }\n    if (!microphonePublication) {\n      return;\n    }\n\n    if (!microphonePublication.subscribed) {\n      microphonePublication.setSubscribed(true);\n    }\n\n    const track = microphonePublication.track;\n    if (track && track !== this.#subscribedTrack) {\n      this.#subscribedTrack = track;\n      if (this.#recognizeTask) {\n        this.#recognizeTask.cancel();\n      }\n\n      const audioStream = new AudioStream(track, 16000);\n\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      this.#recognizeTask = new CancellablePromise(async (resolve, _, onCancel) => {\n        let cancelled = false;\n        onCancel(() => {\n          cancelled = true;\n        });\n\n        const sttStream = this.#stt.stream();\n        const vadStream = this.#vad.stream();\n\n        const audioStreamCo = async () => {\n          for await (const ev of audioStream) {\n            if (cancelled) return;\n            sttStream.pushFrame(ev);\n            vadStream.pushFrame(ev);\n          }\n        };\n\n        const vadStreamCo = async () => {\n          for await (const ev of vadStream) {\n            if (cancelled) return;\n            switch (ev.type) {\n              case VADEventType.START_OF_SPEECH:\n                this.#speaking = true;\n                this.emit(HumanInputEvent.START_OF_SPEECH, ev);\n                break;\n              case VADEventType.INFERENCE_DONE:\n                this.#speechProbability = ev.probability;\n                this.emit(HumanInputEvent.VAD_INFERENCE_DONE, ev);\n                break;\n              case VADEventType.END_OF_SPEECH:\n                this.#speaking = false;\n                this.emit(HumanInputEvent.END_OF_SPEECH, ev);\n                break;\n            }\n          }\n        };\n\n        const sttStreamCo = async () => {\n          for await (const ev of sttStream) {\n            if (cancelled) return;\n            if (ev.type === SpeechEventType.FINAL_TRANSCRIPT) {\n              this.emit(HumanInputEvent.FINAL_TRANSCRIPT, ev);\n            } else if (ev.type == SpeechEventType.INTERIM_TRANSCRIPT) {\n              this.emit(HumanInputEvent.INTERIM_TRANSCRIPT, ev);\n            }\n          }\n        };\n\n        await Promise.all([audioStreamCo(), vadStreamCo(), sttStreamCo()]);\n        sttStream.close();\n        vadStream.close();\n        resolve();\n      });\n    }\n  }\n\n  get speaking(): boolean {\n    return this.#speaking;\n  }\n\n  get speakingProbability(): number {\n    return this.#speechProbability;\n  }\n\n  async close() {\n    if (this.#closed) {\n      throw new Error('HumanInput already closed');\n    }\n    this.#closed = true;\n    this.#room.removeAllListeners();\n    this.#speaking = false;\n    if (this.#recognizeTask) {\n      await gracefullyCancel(this.#recognizeTask);\n    }\n  }\n}\n"],"mappings":"AASA,SAAS,aAAa,WAAW,mBAAmB;AAEpD,SAAS,oBAAoB;AAC7B,SAAS,WAAW;AAEpB,SAAS,uBAAuB;AAChC,SAAS,oBAAoB,wBAAwB;AAErD,SAAS,oBAAoB;AAEtB,IAAK,kBAAL,kBAAKA,qBAAL;AACL,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AACA,EAAAA,kCAAA;AALU,SAAAA;AAAA,GAAA;AAgBL,MAAM,mBAAoB,aAA6D;AAAA,EAC5F,UAAU;AAAA,EACV;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,qBAAqB;AAAA,EACrB,UAAU,IAAI;AAAA,EAEd,YAAY,MAAY,KAAU,KAAU,aAAgC;AAC1E,UAAM;AACN,SAAK,QAAQ;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,eAAe;AAEpB,SAAK,MAAM,GAAG,UAAU,gBAAgB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC9E,SAAK,MAAM,GAAG,UAAU,iBAAiB,KAAK,uBAAuB,KAAK,IAAI,CAAC;AAC/E,SAAK,uBAAuB;AAAA,EAC9B;AAAA,EAEA,yBAA+B;AAC7B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,QAAQ,MAAM,wBAAwB;AAC3C;AAAA,IACF;AAEA,QAAI,wBAA4D;AAChE,eAAW,eAAe,KAAK,aAAa,kBAAkB,OAAO,GAAG;AACtE,UAAI,YAAY,WAAW,YAAY,mBAAmB;AACxD,gCAAwB;AACxB;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,uBAAuB;AAC1B;AAAA,IACF;AAEA,QAAI,CAAC,sBAAsB,YAAY;AACrC,4BAAsB,cAAc,IAAI;AAAA,IAC1C;AAEA,UAAM,QAAQ,sBAAsB;AACpC,QAAI,SAAS,UAAU,KAAK,kBAAkB;AAC5C,WAAK,mBAAmB;AACxB,UAAI,KAAK,gBAAgB;AACvB,aAAK,eAAe,OAAO;AAAA,MAC7B;AAEA,YAAM,cAAc,IAAI,YAAY,OAAO,IAAK;AAGhD,WAAK,iBAAiB,IAAI,mBAAmB,OAAO,SAAS,GAAG,aAAa;AAC3E,YAAI,YAAY;AAChB,iBAAS,MAAM;AACb,sBAAY;AAAA,QACd,CAAC;AAED,cAAM,YAAY,KAAK,KAAK,OAAO;AACnC,cAAM,YAAY,KAAK,KAAK,OAAO;AAEnC,cAAM,gBAAgB,YAAY;AAChC,2BAAiB,MAAM,aAAa;AAClC,gBAAI,UAAW;AACf,sBAAU,UAAU,EAAE;AACtB,sBAAU,UAAU,EAAE;AAAA,UACxB;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,oBAAQ,GAAG,MAAM;AAAA,cACf,KAAK,aAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,yBAAiC,EAAE;AAC7C;AAAA,cACF,KAAK,aAAa;AAChB,qBAAK,qBAAqB,GAAG;AAC7B,qBAAK,KAAK,4BAAoC,EAAE;AAChD;AAAA,cACF,KAAK,aAAa;AAChB,qBAAK,YAAY;AACjB,qBAAK,KAAK,uBAA+B,EAAE;AAC3C;AAAA,YACJ;AAAA,UACF;AAAA,QACF;AAEA,cAAM,cAAc,YAAY;AAC9B,2BAAiB,MAAM,WAAW;AAChC,gBAAI,UAAW;AACf,gBAAI,GAAG,SAAS,gBAAgB,kBAAkB;AAChD,mBAAK,KAAK,0BAAkC,EAAE;AAAA,YAChD,WAAW,GAAG,QAAQ,gBAAgB,oBAAoB;AACxD,mBAAK,KAAK,4BAAoC,EAAE;AAAA,YAClD;AAAA,UACF;AAAA,QACF;AAEA,cAAM,QAAQ,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC,CAAC;AACjE,kBAAU,MAAM;AAChB,kBAAU,MAAM;AAChB,gBAAQ;AAAA,MACV,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,sBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AACA,SAAK,UAAU;AACf,SAAK,MAAM,mBAAmB;AAC9B,SAAK,YAAY;AACjB,QAAI,KAAK,gBAAgB;AACvB,YAAM,iBAAiB,KAAK,cAAc;AAAA,IAC5C;AAAA,EACF;AACF;","names":["HumanInputEvent"]}

package/dist/pipeline/index.cjs

@@ -0,0 +1,31 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var pipeline_exports = {};
+__export(pipeline_exports, {
+  VPAEvent: () => import_pipeline_agent.VPAEvent,
+  VoicePipelineAgent: () => import_pipeline_agent.VoicePipelineAgent
+});
+module.exports = __toCommonJS(pipeline_exports);
+var import_pipeline_agent = require("./pipeline_agent.cjs");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  VPAEvent,
+  VoicePipelineAgent
+});
+//# sourceMappingURL=index.cjs.map

package/dist/pipeline/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/pipeline/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  type AgentState,\n  type BeforeTTSCallback,\n  type BeforeLLMCallback,\n  type VPACallbacks,\n  type AgentCallContext,\n  type AgentTranscriptionOptions,\n  type VPAOptions,\n  VPAEvent,\n  VoicePipelineAgent,\n} from './pipeline_agent.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,4BAUO;","names":[]}

package/dist/pipeline/index.js

@@ -1,5 +1,9 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-export { VPAEvent, VoicePipelineAgent, } from './pipeline_agent.js';
+import {
+  VPAEvent,
+  VoicePipelineAgent
+} from "./pipeline_agent.js";
+export {
+  VPAEvent,
+  VoicePipelineAgent
+};
 //# sourceMappingURL=index.js.map

package/dist/pipeline/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AAEtC,OAAO,EAQL,QAAQ,EACR,kBAAkB,GACnB,MAAM,qBAAqB,CAAC"}
+{"version":3,"sources":["../../src/pipeline/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  type AgentState,\n  type BeforeTTSCallback,\n  type BeforeLLMCallback,\n  type VPACallbacks,\n  type AgentCallContext,\n  type AgentTranscriptionOptions,\n  type VPAOptions,\n  VPAEvent,\n  VoicePipelineAgent,\n} from './pipeline_agent.js';\n"],"mappings":"AAIA;AAAA,EAQE;AAAA,EACA;AAAA,OACK;","names":[]}