@livekit/agents 0.7.9 → 1.0.0-next.1

package/dist/voice/audio_recognition.js

@@ -0,0 +1,350 @@
+import { AudioFrame } from "@livekit/rtc-node";
+import { ReadableStream } from "node:stream/web";
+import {} from "../llm/chat_context.js";
+import { log } from "../log.js";
+import { DeferredReadableStream, isStreamReaderReleaseError } from "../stream/deferred_stream.js";
+import { IdentityTransform } from "../stream/identity_transform.js";
+import { mergeReadableStreams } from "../stream/merge_readable_streams.js";
+import { SpeechEventType } from "../stt/stt.js";
+import { Task, delay } from "../utils.js";
+import { VADEventType } from "../vad.js";
+class AudioRecognition {
+  hooks;
+  stt;
+  vad;
+  turnDetector;
+  turnDetectionMode;
+  minEndpointingDelay;
+  maxEndpointingDelay;
+  lastLanguage;
+  deferredInputStream;
+  logger = log();
+  lastFinalTranscriptTime = 0;
+  audioTranscript = "";
+  audioInterimTranscript = "";
+  lastSpeakingTime = 0;
+  userTurnCommitted = false;
+  speaking = false;
+  sampleRate;
+  vadInputStream;
+  sttInputStream;
+  silenceAudioTransform = new IdentityTransform();
+  silenceAudioWriter;
+  // all cancellable tasks
+  bounceEOUTask;
+  commitUserTurnTask;
+  vadTask;
+  sttTask;
+  constructor(opts) {
+    this.hooks = opts.recognitionHooks;
+    this.stt = opts.stt;
+    this.vad = opts.vad;
+    this.turnDetector = opts.turnDetector;
+    this.turnDetectionMode = opts.turnDetectionMode;
+    this.minEndpointingDelay = opts.minEndpointingDelay;
+    this.maxEndpointingDelay = opts.maxEndpointingDelay;
+    this.lastLanguage = void 0;
+    this.deferredInputStream = new DeferredReadableStream();
+    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
+    this.vadInputStream = vadInputStream;
+    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);
+    this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();
+  }
+  /**
+   * Current transcript of the user's speech, including interim transcript if available.
+   */
+  get currentTranscript() {
+    if (this.audioInterimTranscript) {
+      return `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();
+    }
+    return this.audioTranscript;
+  }
+  async start() {
+    this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));
+    this.vadTask.result.catch((err) => {
+      this.logger.error(`Error running VAD task: ${err}`);
+    });
+    this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));
+    this.sttTask.result.catch((err) => {
+      this.logger.error(`Error running STT task: ${err}`);
+    });
+  }
+  async onSTTEvent(ev) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
+    if (this.turnDetectionMode === "manual" && this.userTurnCommitted && (this.bounceEOUTask === void 0 || this.bounceEOUTask.done || ev.type == SpeechEventType.INTERIM_TRANSCRIPT)) {
+      this.logger.debug(
+        {
+          userTurnCommitted: this.userTurnCommitted,
+          eouTaskDone: (_a = this.bounceEOUTask) == null ? void 0 : _a.done,
+          evType: ev.type,
+          turnDetectionMode: this.turnDetectionMode
+        },
+        "ignoring stt event"
+      );
+      return;
+    }
+    switch (ev.type) {
+      case SpeechEventType.FINAL_TRANSCRIPT:
+        this.hooks.onFinalTranscript(ev);
+        const transcript = (_c = (_b = ev.alternatives) == null ? void 0 : _b[0]) == null ? void 0 : _c.text;
+        this.lastLanguage = (_e = (_d = ev.alternatives) == null ? void 0 : _d[0]) == null ? void 0 : _e.language;
+        if (!transcript) {
+          return;
+        }
+        this.logger.debug(
+          {
+            user_transcript: transcript,
+            language: this.lastLanguage
+          },
+          "received user transcript"
+        );
+        this.lastFinalTranscriptTime = Date.now();
+        this.audioTranscript += ` ${transcript}`;
+        this.audioTranscript = this.audioTranscript.trimStart();
+        this.audioInterimTranscript = "";
+        if (!this.speaking) {
+          if (!this.vad) {
+            this.lastSpeakingTime = Date.now();
+          }
+          if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+            const chatCtx = this.hooks.retrieveChatCtx();
+            this.logger.debug("running EOU detection on stt FINAL_TRANSCRIPT");
+            this.runEOUDetection(chatCtx);
+          }
+        }
+        break;
+      case SpeechEventType.INTERIM_TRANSCRIPT:
+        this.logger.debug({ transcript: (_g = (_f = ev.alternatives) == null ? void 0 : _f[0]) == null ? void 0 : _g.text }, "interim transcript");
+        this.hooks.onInterimTranscript(ev);
+        this.audioInterimTranscript = ((_i = (_h = ev.alternatives) == null ? void 0 : _h[0]) == null ? void 0 : _i.text) ?? "";
+        break;
+      case SpeechEventType.END_OF_SPEECH:
+        if (this.turnDetectionMode !== "stt") break;
+        this.userTurnCommitted = true;
+        if (!this.speaking) {
+          const chatCtx = this.hooks.retrieveChatCtx();
+          this.logger.debug("running EOU detection on stt END_OF_SPEECH");
+          this.runEOUDetection(chatCtx);
+        }
+    }
+  }
+  runEOUDetection(chatCtx) {
+    var _a;
+    this.logger.debug(
+      {
+        stt: this.stt,
+        audioTranscript: this.audioTranscript,
+        turnDetectionMode: this.turnDetectionMode
+      },
+      "running EOU detection"
+    );
+    if (this.stt && !this.audioTranscript && this.turnDetectionMode !== "manual") {
+      this.logger.debug("skipping EOU detection");
+      return;
+    }
+    chatCtx = chatCtx.copy();
+    chatCtx.addMessage({ role: "user", content: this.audioTranscript });
+    const turnDetector = (
+      // disable EOU model if manual turn detection enabled
+      this.audioTranscript && this.turnDetectionMode !== "manual" ? this.turnDetector : void 0
+    );
+    const bounceEOUTask = (lastSpeakingTime) => async (controller) => {
+      let endpointingDelay = this.minEndpointingDelay;
+      if (turnDetector) {
+        this.logger.debug("Running turn detector model");
+        if (!turnDetector.supportsLanguage(this.lastLanguage)) {
+          this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);
+        } else {
+          const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);
+          this.logger.debug(
+            { endOfTurnProbability, language: this.lastLanguage },
+            "end of turn probability"
+          );
+          const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);
+          this.logger.debug(
+            {
+              unlikelyThreshold,
+              endOfTurnProbability,
+              language: this.lastLanguage,
+              transcript: this.audioTranscript
+            },
+            "EOU Detection"
+          );
+          if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {
+            endpointingDelay = this.maxEndpointingDelay;
+          }
+        }
+      }
+      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();
+      await delay(Math.max(extraSleep, 0), { signal: controller.signal });
+      this.logger.debug({ transcript: this.audioTranscript }, "end of user turn");
+      const committed = await this.hooks.onEndOfTurn({
+        newTranscript: this.audioTranscript,
+        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),
+        endOfUtteranceDelay: Date.now() - lastSpeakingTime
+      });
+      if (committed) {
+        this.audioTranscript = "";
+      }
+      this.userTurnCommitted = false;
+    };
+    (_a = this.bounceEOUTask) == null ? void 0 : _a.cancel();
+    this.bounceEOUTask = Task.from(bounceEOUTask(this.lastSpeakingTime));
+    this.bounceEOUTask.result.then(() => {
+      this.logger.debug("EOU detection task completed");
+    }).catch((err) => {
+      if (err instanceof Error && err.message.includes("This operation was aborted")) {
+        return;
+      }
+      this.logger.error(err, "Error in EOU detection task:");
+    });
+  }
+  async createSttTask(stt, signal) {
+    if (!stt) return;
+    this.logger.debug("createSttTask: create stt stream from stt node");
+    const sttStream = await stt(this.sttInputStream, {});
+    if (signal.aborted || sttStream === null) return;
+    if (sttStream instanceof ReadableStream) {
+      const reader = sttStream.getReader();
+      signal.addEventListener("abort", async () => {
+        try {
+          reader.releaseLock();
+          await (sttStream == null ? void 0 : sttStream.cancel());
+        } catch (e) {
+          this.logger.debug("createSttTask: error during abort handler:", e);
+        }
+      });
+      try {
+        while (true) {
+          if (signal.aborted) break;
+          const { done, value: ev } = await reader.read();
+          if (done) break;
+          if (typeof ev === "string") {
+            throw new Error("STT node must yield SpeechEvent");
+          } else {
+            await this.onSTTEvent(ev);
+          }
+        }
+      } catch (e) {
+        if (isStreamReaderReleaseError(e)) {
+          return;
+        }
+        this.logger.error({ error: e }, "createSttTask: error reading sttStream");
+      } finally {
+        reader.releaseLock();
+        try {
+          await sttStream.cancel();
+        } catch (e) {
+          this.logger.debug(
+            "createSttTask: error cancelling sttStream (may already be cancelled):",
+            e
+          );
+        }
+      }
+    }
+  }
+  async createVadTask(vad, signal) {
+    var _a;
+    if (!vad) return;
+    const vadStream = vad.stream();
+    vadStream.updateInputStream(this.vadInputStream);
+    const abortHandler = () => {
+      vadStream.detachInputStream();
+      vadStream.close();
+      signal.removeEventListener("abort", abortHandler);
+    };
+    signal.addEventListener("abort", abortHandler);
+    try {
+      for await (const ev of vadStream) {
+        if (signal.aborted) break;
+        switch (ev.type) {
+          case VADEventType.START_OF_SPEECH:
+            this.logger.debug("VAD task: START_OF_SPEECH");
+            this.hooks.onStartOfSpeech(ev);
+            this.speaking = true;
+            (_a = this.bounceEOUTask) == null ? void 0 : _a.cancel();
+            break;
+          case VADEventType.INFERENCE_DONE:
+            this.hooks.onVADInferenceDone(ev);
+            break;
+          case VADEventType.END_OF_SPEECH:
+            this.logger.debug("VAD task: END_OF_SPEECH");
+            this.hooks.onEndOfSpeech(ev);
+            this.speaking = false;
+            this.lastSpeakingTime = Date.now() - ev.silenceDuration;
+            if (this.vadBaseTurnDetection || this.turnDetectionMode === "stt" && this.userTurnCommitted) {
+              const chatCtx = this.hooks.retrieveChatCtx();
+              this.runEOUDetection(chatCtx);
+            }
+            break;
+        }
+      }
+    } catch (e) {
+      this.logger.error(e, "Error in VAD task");
+    } finally {
+      this.logger.debug("VAD task closed");
+    }
+  }
+  setInputAudioStream(audioStream) {
+    this.deferredInputStream.setSource(audioStream);
+  }
+  detachInputAudioStream() {
+    this.deferredInputStream.detachSource();
+  }
+  clearUserTurn() {
+    var _a;
+    this.audioTranscript = "";
+    this.audioInterimTranscript = "";
+    this.userTurnCommitted = false;
+    (_a = this.sttTask) == null ? void 0 : _a.cancelAndWait().finally(() => {
+      this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));
+      this.sttTask.result.catch((err) => {
+        this.logger.error(`Error running STT task: ${err}`);
+      });
+    });
+  }
+  commitUserTurn(audioDetached) {
+    var _a;
+    const commitUserTurnTask = (delayDuration = 500) => async (controller) => {
+      if (Date.now() - this.lastFinalTranscriptTime > delayDuration) {
+        if (audioDetached && this.sampleRate !== void 0) {
+          const numSamples = Math.floor(this.sampleRate * 0.5);
+          const silence = new Int16Array(numSamples * 2);
+          const silenceFrame = new AudioFrame(silence, this.sampleRate, 1, numSamples);
+          this.silenceAudioWriter.write(silenceFrame);
+        }
+        await delay(delayDuration, { signal: controller.signal });
+      }
+      if (this.audioInterimTranscript) {
+        this.audioTranscript = `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();
+      }
+      this.audioInterimTranscript = "";
+      const chatCtx = this.hooks.retrieveChatCtx();
+      this.logger.debug("running EOU detection on commitUserTurn");
+      this.runEOUDetection(chatCtx);
+      this.userTurnCommitted = true;
+    };
+    (_a = this.commitUserTurnTask) == null ? void 0 : _a.cancel();
+    this.commitUserTurnTask = Task.from(commitUserTurnTask());
+    this.commitUserTurnTask.result.then(() => {
+      this.logger.debug("User turn committed");
+    }).catch((err) => {
+      this.logger.error(err, "Error in user turn commit task:");
+    });
+  }
+  async close() {
+    var _a, _b, _c, _d;
+    this.detachInputAudioStream();
+    await ((_a = this.commitUserTurnTask) == null ? void 0 : _a.cancelAndWait());
+    await ((_b = this.sttTask) == null ? void 0 : _b.cancelAndWait());
+    await ((_c = this.vadTask) == null ? void 0 : _c.cancelAndWait());
+    await ((_d = this.bounceEOUTask) == null ? void 0 : _d.cancelAndWait());
+  }
+  get vadBaseTurnDetection() {
+    return ["vad", void 0].includes(this.turnDetectionMode);
+  }
+}
+export {
+  AudioRecognition
+};
+//# sourceMappingURL=audio_recognition.js.map

package/dist/voice/audio_recognition.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/voice/audio_recognition.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { WritableStreamDefaultWriter } from 'node:stream/web';\nimport { ReadableStream } from 'node:stream/web';\nimport { type ChatContext } from '../llm/chat_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream, isStreamReaderReleaseError } from '../stream/deferred_stream.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { mergeReadableStreams } from '../stream/merge_readable_streams.js';\nimport { type SpeechEvent, SpeechEventType } from '../stt/stt.js';\nimport { Task, delay } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { TurnDetectionMode } from './agent_session.js';\nimport type { STTNode } from './io.js';\n\nexport interface EndOfTurnInfo {\n  newTranscript: string;\n  transcriptionDelay: number;\n  endOfUtteranceDelay: number;\n}\n\nexport interface RecognitionHooks {\n  onStartOfSpeech: (ev: VADEvent) => void;\n  onVADInferenceDone: (ev: VADEvent) => void;\n  onEndOfSpeech: (ev: VADEvent) => void;\n  onInterimTranscript: (ev: SpeechEvent) => void;\n  onFinalTranscript: (ev: SpeechEvent) => void;\n  onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;\n\n  retrieveChatCtx: () => ChatContext;\n}\n\nexport interface _TurnDetector {\n  unlikelyThreshold: (language?: string) => Promise<number | undefined>;\n  supportsLanguage: (language?: string) => Promise<boolean>;\n  predictEndOfTurn(chatCtx: ChatContext): Promise<number>;\n}\n\nexport interface AudioRecognitionOptions {\n  recognitionHooks: RecognitionHooks;\n  stt?: STTNode;\n  vad?: VAD;\n  turnDetector?: _TurnDetector;\n  turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n}\n\nexport class AudioRecognition {\n  private hooks: RecognitionHooks;\n  private stt?: STTNode;\n  private vad?: VAD;\n  private turnDetector?: _TurnDetector;\n  private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  private minEndpointingDelay: number;\n  private maxEndpointingDelay: number;\n  private lastLanguage?: string;\n\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private lastFinalTranscriptTime = 0;\n  private audioTranscript = '';\n  private audioInterimTranscript = '';\n  private lastSpeakingTime = 0;\n  private userTurnCommitted = false;\n  private speaking = false;\n  private sampleRate?: number;\n\n  private vadInputStream: ReadableStream<AudioFrame>;\n  private sttInputStream: ReadableStream<AudioFrame>;\n  private silenceAudioTransform = new IdentityTransform<AudioFrame>();\n  private silenceAudioWriter: WritableStreamDefaultWriter<AudioFrame>;\n\n  // all cancellable tasks\n  private bounceEOUTask?: Task<void>;\n  private commitUserTurnTask?: Task<void>;\n  private vadTask?: Task<void>;\n  private sttTask?: Task<void>;\n\n  constructor(opts: AudioRecognitionOptions) {\n    this.hooks = opts.recognitionHooks;\n    this.stt = opts.stt;\n    this.vad = opts.vad;\n    this.turnDetector = opts.turnDetector;\n    this.turnDetectionMode = opts.turnDetectionMode;\n    this.minEndpointingDelay = opts.minEndpointingDelay;\n    this.maxEndpointingDelay = opts.maxEndpointingDelay;\n    this.lastLanguage = undefined;\n\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();\n    this.vadInputStream = vadInputStream;\n    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);\n    this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();\n  }\n\n  /**\n   * Current transcript of the user's speech, including interim transcript if available.\n   */\n  get currentTranscript(): string {\n    if (this.audioInterimTranscript) {\n      return `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n    }\n    return this.audioTranscript;\n  }\n\n  async start() {\n    this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));\n    this.vadTask.result.catch((err) => {\n      this.logger.error(`Error running VAD task: ${err}`);\n    });\n\n    this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n    this.sttTask.result.catch((err) => {\n      this.logger.error(`Error running STT task: ${err}`);\n    });\n  }\n\n  private async onSTTEvent(ev: SpeechEvent) {\n    if (\n      this.turnDetectionMode === 'manual' &&\n      this.userTurnCommitted &&\n      (this.bounceEOUTask === undefined ||\n        this.bounceEOUTask.done ||\n        ev.type == SpeechEventType.INTERIM_TRANSCRIPT)\n    ) {\n      // ignore stt event if user turn already committed and EOU task is done\n      // or it's an interim transcript\n      this.logger.debug(\n        {\n          userTurnCommitted: this.userTurnCommitted,\n          eouTaskDone: this.bounceEOUTask?.done,\n          evType: ev.type,\n          turnDetectionMode: this.turnDetectionMode,\n        },\n        'ignoring stt event',\n      );\n      return;\n    }\n\n    switch (ev.type) {\n      case SpeechEventType.FINAL_TRANSCRIPT:\n        this.hooks.onFinalTranscript(ev);\n        const transcript = ev.alternatives?.[0]?.text;\n        this.lastLanguage = ev.alternatives?.[0]?.language;\n\n        if (!transcript) {\n          // stt final transcript received but no transcript\n          return;\n        }\n\n        this.logger.debug(\n          {\n            user_transcript: transcript,\n            language: this.lastLanguage,\n          },\n          'received user transcript',\n        );\n\n        this.lastFinalTranscriptTime = Date.now();\n        this.audioTranscript += ` ${transcript}`;\n        this.audioTranscript = this.audioTranscript.trimStart();\n        this.audioInterimTranscript = '';\n\n        if (!this.speaking) {\n          if (!this.vad) {\n            // Copied from python agents:\n            // vad disabled, use stt timestamp\n            // TODO: this would screw up transcription latency metrics\n            // but we'll live with it for now.\n            // the correct way is to ensure STT fires SpeechEventType.END_OF_SPEECH\n            // and using that timestamp for _last_speaking_time\n            this.lastSpeakingTime = Date.now();\n          }\n\n          if (this.vadBaseTurnDetection || this.userTurnCommitted) {\n            const chatCtx = this.hooks.retrieveChatCtx();\n            this.logger.debug('running EOU detection on stt FINAL_TRANSCRIPT');\n            this.runEOUDetection(chatCtx);\n          }\n        }\n        break;\n      case SpeechEventType.INTERIM_TRANSCRIPT:\n        this.logger.debug({ transcript: ev.alternatives?.[0]?.text }, 'interim transcript');\n        this.hooks.onInterimTranscript(ev);\n        this.audioInterimTranscript = ev.alternatives?.[0]?.text ?? '';\n        break;\n      case SpeechEventType.END_OF_SPEECH:\n        if (this.turnDetectionMode !== 'stt') break;\n        this.userTurnCommitted = true;\n\n        if (!this.speaking) {\n          const chatCtx = this.hooks.retrieveChatCtx();\n          this.logger.debug('running EOU detection on stt END_OF_SPEECH');\n          this.runEOUDetection(chatCtx);\n        }\n    }\n  }\n\n  private runEOUDetection(chatCtx: ChatContext) {\n    this.logger.debug(\n      {\n        stt: this.stt,\n        audioTranscript: this.audioTranscript,\n        turnDetectionMode: this.turnDetectionMode,\n      },\n      'running EOU detection',\n    );\n\n    if (this.stt && !this.audioTranscript && this.turnDetectionMode !== 'manual') {\n      // stt enabled but no transcript yet\n      this.logger.debug('skipping EOU detection');\n      return;\n    }\n\n    chatCtx = chatCtx.copy();\n    chatCtx.addMessage({ role: 'user', content: this.audioTranscript });\n\n    const turnDetector =\n      // disable EOU model if manual turn detection enabled\n      this.audioTranscript && this.turnDetectionMode !== 'manual' ? this.turnDetector : undefined;\n\n    const bounceEOUTask = (lastSpeakingTime: number) => async (controller: AbortController) => {\n      let endpointingDelay = this.minEndpointingDelay;\n\n      // TODO(AJS-74): need to support actual turn detection model plugins for following code to run\n      if (turnDetector) {\n        this.logger.debug('Running turn detector model');\n        if (!turnDetector.supportsLanguage(this.lastLanguage)) {\n          this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);\n        } else {\n          const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);\n          this.logger.debug(\n            { endOfTurnProbability, language: this.lastLanguage },\n            'end of turn probability',\n          );\n\n          const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);\n          this.logger.debug(\n            {\n              unlikelyThreshold,\n              endOfTurnProbability,\n              language: this.lastLanguage,\n              transcript: this.audioTranscript,\n            },\n            'EOU Detection',\n          );\n\n          if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {\n            endpointingDelay = this.maxEndpointingDelay;\n          }\n        }\n      }\n\n      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();\n      // add delay to see if there's a potential upcoming EOU task that cancels this one\n      await delay(Math.max(extraSleep, 0), { signal: controller.signal });\n\n      this.logger.debug({ transcript: this.audioTranscript }, 'end of user turn');\n\n      const committed = await this.hooks.onEndOfTurn({\n        newTranscript: this.audioTranscript,\n        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),\n        endOfUtteranceDelay: Date.now() - lastSpeakingTime,\n      });\n\n      if (committed) {\n        // clear the transcript if the user turn was committed\n        this.audioTranscript = '';\n      }\n\n      this.userTurnCommitted = false;\n    };\n\n    // cancel any existing EOU task\n    this.bounceEOUTask?.cancel();\n    this.bounceEOUTask = Task.from(bounceEOUTask(this.lastSpeakingTime));\n\n    this.bounceEOUTask.result\n      .then(() => {\n        this.logger.debug('EOU detection task completed');\n      })\n      .catch((err: unknown) => {\n        if (err instanceof Error && err.message.includes('This operation was aborted')) {\n          // ignore aborted errors\n          return;\n        }\n        this.logger.error(err, 'Error in EOU detection task:');\n      });\n  }\n\n  private async createSttTask(stt: STTNode | undefined, signal: AbortSignal) {\n    if (!stt) return;\n\n    this.logger.debug('createSttTask: create stt stream from stt node');\n\n    const sttStream = await stt(this.sttInputStream, {});\n\n    if (signal.aborted || sttStream === null) return;\n\n    if (sttStream instanceof ReadableStream) {\n      const reader = sttStream.getReader();\n\n      signal.addEventListener('abort', async () => {\n        try {\n          reader.releaseLock();\n          await sttStream?.cancel();\n        } catch (e) {\n          this.logger.debug('createSttTask: error during abort handler:', e);\n        }\n      });\n\n      try {\n        while (true) {\n          if (signal.aborted) break;\n\n          const { done, value: ev } = await reader.read();\n          if (done) break;\n\n          if (typeof ev === 'string') {\n            throw new Error('STT node must yield SpeechEvent');\n          } else {\n            await this.onSTTEvent(ev);\n          }\n        }\n      } catch (e) {\n        if (isStreamReaderReleaseError(e)) {\n          return;\n        }\n        this.logger.error({ error: e }, 'createSttTask: error reading sttStream');\n      } finally {\n        reader.releaseLock();\n        try {\n          await sttStream.cancel();\n        } catch (e) {\n          this.logger.debug(\n            'createSttTask: error cancelling sttStream (may already be cancelled):',\n            e,\n          );\n        }\n      }\n    }\n  }\n\n  private async createVadTask(vad: VAD | undefined, signal: AbortSignal) {\n    if (!vad) return;\n\n    const vadStream = vad.stream();\n    vadStream.updateInputStream(this.vadInputStream);\n\n    const abortHandler = () => {\n      vadStream.detachInputStream();\n      vadStream.close();\n      signal.removeEventListener('abort', abortHandler);\n    };\n    signal.addEventListener('abort', abortHandler);\n\n    try {\n      for await (const ev of vadStream) {\n        if (signal.aborted) break;\n\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.logger.debug('VAD task: START_OF_SPEECH');\n            this.hooks.onStartOfSpeech(ev);\n            this.speaking = true;\n\n            this.bounceEOUTask?.cancel();\n            break;\n          case VADEventType.INFERENCE_DONE:\n            this.hooks.onVADInferenceDone(ev);\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.logger.debug('VAD task: END_OF_SPEECH');\n            this.hooks.onEndOfSpeech(ev);\n            this.speaking = false;\n            // when VAD fires END_OF_SPEECH, it already waited for the silence_duration\n            this.lastSpeakingTime = Date.now() - ev.silenceDuration;\n\n            if (\n              this.vadBaseTurnDetection ||\n              (this.turnDetectionMode === 'stt' && this.userTurnCommitted)\n            ) {\n              const chatCtx = this.hooks.retrieveChatCtx();\n              this.runEOUDetection(chatCtx);\n            }\n            break;\n        }\n      }\n    } catch (e) {\n      this.logger.error(e, 'Error in VAD task');\n    } finally {\n      this.logger.debug('VAD task closed');\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputAudioStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  clearUserTurn() {\n    this.audioTranscript = '';\n    this.audioInterimTranscript = '';\n    this.userTurnCommitted = false;\n\n    this.sttTask?.cancelAndWait().finally(() => {\n      this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n      this.sttTask.result.catch((err) => {\n        this.logger.error(`Error running STT task: ${err}`);\n      });\n    });\n  }\n\n  commitUserTurn(audioDetached: boolean) {\n    const commitUserTurnTask =\n      (delayDuration: number = 500) =>\n      async (controller: AbortController) => {\n        if (Date.now() - this.lastFinalTranscriptTime > delayDuration) {\n          // flush the stt by pushing silence\n          if (audioDetached && this.sampleRate !== undefined) {\n            const numSamples = Math.floor(this.sampleRate * 0.5);\n            const silence = new Int16Array(numSamples * 2);\n            const silenceFrame = new AudioFrame(silence, this.sampleRate, 1, numSamples);\n            this.silenceAudioWriter.write(silenceFrame);\n          }\n\n          // wait for the final transcript to be available\n          await delay(delayDuration, { signal: controller.signal });\n        }\n\n        if (this.audioInterimTranscript) {\n          // append interim transcript in case the final transcript is not ready\n          this.audioTranscript = `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n        }\n        this.audioInterimTranscript = '';\n\n        const chatCtx = this.hooks.retrieveChatCtx();\n        this.logger.debug('running EOU detection on commitUserTurn');\n        this.runEOUDetection(chatCtx);\n        this.userTurnCommitted = true;\n      };\n\n    // cancel any existing commit user turn task\n    this.commitUserTurnTask?.cancel();\n    this.commitUserTurnTask = Task.from(commitUserTurnTask());\n\n    this.commitUserTurnTask.result\n      .then(() => {\n        this.logger.debug('User turn committed');\n      })\n      .catch((err: unknown) => {\n        this.logger.error(err, 'Error in user turn commit task:');\n      });\n  }\n\n  async close() {\n    this.detachInputAudioStream();\n    await this.commitUserTurnTask?.cancelAndWait();\n    await this.sttTask?.cancelAndWait();\n    await this.vadTask?.cancelAndWait();\n    await this.bounceEOUTask?.cancelAndWait();\n  }\n\n  private get vadBaseTurnDetection() {\n    return ['vad', undefined].includes(this.turnDetectionMode);\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAE3B,SAAS,sBAAsB;AAC/B,eAAiC;AACjC,SAAS,WAAW;AACpB,SAAS,wBAAwB,kCAAkC;AACnE,SAAS,yBAAyB;AAClC,SAAS,4BAA4B;AACrC,SAA2B,uBAAuB;AAClD,SAAS,MAAM,aAAa;AAC5B,SAAkC,oBAAoB;AAqC/C,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA,SAAS,IAAI;AAAA,EACb,0BAA0B;AAAA,EAC1B,kBAAkB;AAAA,EAClB,yBAAyB;AAAA,EACzB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,WAAW;AAAA,EACX;AAAA,EAEA;AAAA,EACA;AAAA,EACA,wBAAwB,IAAI,kBAA8B;AAAA,EAC1D;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,MAA+B;AACzC,SAAK,QAAQ,KAAK;AAClB,SAAK,MAAM,KAAK;AAChB,SAAK,MAAM,KAAK;AAChB,SAAK,eAAe,KAAK;AACzB,SAAK,oBAAoB,KAAK;AAC9B,SAAK,sBAAsB,KAAK;AAChC,SAAK,sBAAsB,KAAK;AAChC,SAAK,eAAe;AAEpB,SAAK,sBAAsB,IAAI,uBAAmC;AAClE,UAAM,CAAC,gBAAgB,cAAc,IAAI,KAAK,oBAAoB,OAAO,IAAI;AAC7E,SAAK,iBAAiB;AACtB,SAAK,iBAAiB,qBAAqB,gBAAgB,KAAK,sBAAsB,QAAQ;AAC9F,SAAK,qBAAqB,KAAK,sBAAsB,SAAS,UAAU;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,oBAA4B;AAC9B,QAAI,KAAK,wBAAwB;AAC/B,aAAO,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,IACvE;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAED,SAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,WAAW,IAAiB;AAxH5C;AAyHI,QACE,KAAK,sBAAsB,YAC3B,KAAK,sBACJ,KAAK,kBAAkB,UACtB,KAAK,cAAc,QACnB,GAAG,QAAQ,gBAAgB,qBAC7B;AAGA,WAAK,OAAO;AAAA,QACV;AAAA,UACE,mBAAmB,KAAK;AAAA,UACxB,cAAa,UAAK,kBAAL,mBAAoB;AAAA,UACjC,QAAQ,GAAG;AAAA,UACX,mBAAmB,KAAK;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AACA;AAAA,IACF;AAEA,YAAQ,GAAG,MAAM;AAAA,MACf,KAAK,gBAAgB;AACnB,aAAK,MAAM,kBAAkB,EAAE;AAC/B,cAAM,cAAa,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AACzC,aAAK,gBAAe,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AAE1C,YAAI,CAAC,YAAY;AAEf;AAAA,QACF;AAEA,aAAK,OAAO;AAAA,UACV;AAAA,YACE,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,UACjB;AAAA,UACA;AAAA,QACF;AAEA,aAAK,0BAA0B,KAAK,IAAI;AACxC,aAAK,mBAAmB,IAAI,UAAU;AACtC,aAAK,kBAAkB,KAAK,gBAAgB,UAAU;AACtD,aAAK,yBAAyB;AAE9B,YAAI,CAAC,KAAK,UAAU;AAClB,cAAI,CAAC,KAAK,KAAK;AAOb,iBAAK,mBAAmB,KAAK,IAAI;AAAA,UACnC;AAEA,cAAI,KAAK,wBAAwB,KAAK,mBAAmB;AACvD,kBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,iBAAK,OAAO,MAAM,+CAA+C;AACjE,iBAAK,gBAAgB,OAAO;AAAA,UAC9B;AAAA,QACF;AACA;AAAA,MACF,KAAK,gBAAgB;AACnB,aAAK,OAAO,MAAM,EAAE,aAAY,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,KAAK,GAAG,oBAAoB;AAClF,aAAK,MAAM,oBAAoB,EAAE;AACjC,aAAK,2BAAyB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,SAAQ;AAC5D;AAAA,MACF,KAAK,gBAAgB;AACnB,YAAI,KAAK,sBAAsB,MAAO;AACtC,aAAK,oBAAoB;AAEzB,YAAI,CAAC,KAAK,UAAU;AAClB,gBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,eAAK,OAAO,MAAM,4CAA4C;AAC9D,eAAK,gBAAgB,OAAO;AAAA,QAC9B;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,gBAAgB,SAAsB;AAzMhD;AA0MI,SAAK,OAAO;AAAA,MACV;AAAA,QACE,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,mBAAmB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,CAAC,KAAK,mBAAmB,KAAK,sBAAsB,UAAU;AAE5E,WAAK,OAAO,MAAM,wBAAwB;AAC1C;AAAA,IACF;AAEA,cAAU,QAAQ,KAAK;AACvB,YAAQ,WAAW,EAAE,MAAM,QAAQ,SAAS,KAAK,gBAAgB,CAAC;AAElE,UAAM;AAAA;AAAA,MAEJ,KAAK,mBAAmB,KAAK,sBAAsB,WAAW,KAAK,eAAe;AAAA;AAEpF,UAAM,gBAAgB,CAAC,qBAA6B,OAAO,eAAgC;AACzF,UAAI,mBAAmB,KAAK;AAG5B,UAAI,cAAc;AAChB,aAAK,OAAO,MAAM,6BAA6B;AAC/C,YAAI,CAAC,aAAa,iBAAiB,KAAK,YAAY,GAAG;AACrD,eAAK,OAAO,MAAM,2CAA2C,KAAK,YAAY,EAAE;AAAA,QAClF,OAAO;AACL,gBAAM,uBAAuB,MAAM,aAAa,iBAAiB,OAAO;AACxE,eAAK,OAAO;AAAA,YACV,EAAE,sBAAsB,UAAU,KAAK,aAAa;AAAA,YACpD;AAAA,UACF;AAEA,gBAAM,oBAAoB,MAAM,aAAa,kBAAkB,KAAK,YAAY;AAChF,eAAK,OAAO;AAAA,YACV;AAAA,cACE;AAAA,cACA;AAAA,cACA,UAAU,KAAK;AAAA,cACf,YAAY,KAAK;AAAA,YACnB;AAAA,YACA;AAAA,UACF;AAEA,cAAI,qBAAqB,uBAAuB,mBAAmB;AACjE,+BAAmB,KAAK;AAAA,UAC1B;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,mBAAmB,mBAAmB,KAAK,IAAI;AAElE,YAAM,MAAM,KAAK,IAAI,YAAY,CAAC,GAAG,EAAE,QAAQ,WAAW,OAAO,CAAC;AAElE,WAAK,OAAO,MAAM,EAAE,YAAY,KAAK,gBAAgB,GAAG,kBAAkB;AAE1E,YAAM,YAAY,MAAM,KAAK,MAAM,YAAY;AAAA,QAC7C,eAAe,KAAK;AAAA,QACpB,oBAAoB,KAAK,IAAI,KAAK,0BAA0B,kBAAkB,CAAC;AAAA,QAC/E,qBAAqB,KAAK,IAAI,IAAI;AAAA,MACpC,CAAC;AAED,UAAI,WAAW;AAEb,aAAK,kBAAkB;AAAA,MACzB;AAEA,WAAK,oBAAoB;AAAA,IAC3B;AAGA,eAAK,kBAAL,mBAAoB;AACpB,SAAK,gBAAgB,KAAK,KAAK,cAAc,KAAK,gBAAgB,CAAC;AAEnE,SAAK,cAAc,OAChB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,8BAA8B;AAAA,IAClD,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,UAAI,eAAe,SAAS,IAAI,QAAQ,SAAS,4BAA4B,GAAG;AAE9E;AAAA,MACF;AACA,WAAK,OAAO,MAAM,KAAK,8BAA8B;AAAA,IACvD,CAAC;AAAA,EACL;AAAA,EAEA,MAAc,cAAc,KAA0B,QAAqB;AACzE,QAAI,CAAC,IAAK;AAEV,SAAK,OAAO,MAAM,gDAAgD;AAElE,UAAM,YAAY,MAAM,IAAI,KAAK,gBAAgB,CAAC,CAAC;AAEnD,QAAI,OAAO,WAAW,cAAc,KAAM;AAE1C,QAAI,qBAAqB,gBAAgB;AACvC,YAAM,SAAS,UAAU,UAAU;AAEnC,aAAO,iBAAiB,SAAS,YAAY;AAC3C,YAAI;AACF,iBAAO,YAAY;AACnB,iBAAM,uCAAW;AAAA,QACnB,SAAS,GAAG;AACV,eAAK,OAAO,MAAM,8CAA8C,CAAC;AAAA,QACnE;AAAA,MACF,CAAC;AAED,UAAI;AACF,eAAO,MAAM;AACX,cAAI,OAAO,QAAS;AAEpB,gBAAM,EAAE,MAAM,OAAO,GAAG,IAAI,MAAM,OAAO,KAAK;AAC9C,cAAI,KAAM;AAEV,cAAI,OAAO,OAAO,UAAU;AAC1B,kBAAM,IAAI,MAAM,iCAAiC;AAAA,UACnD,OAAO;AACL,kBAAM,KAAK,WAAW,EAAE;AAAA,UAC1B;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,YAAI,2BAA2B,CAAC,GAAG;AACjC;AAAA,QACF;AACA,aAAK,OAAO,MAAM,EAAE,OAAO,EAAE,GAAG,wCAAwC;AAAA,MAC1E,UAAE;AACA,eAAO,YAAY;AACnB,YAAI;AACF,gBAAM,UAAU,OAAO;AAAA,QACzB,SAAS,GAAG;AACV,eAAK,OAAO;AAAA,YACV;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,KAAsB,QAAqB;AA1VzE;AA2VI,QAAI,CAAC,IAAK;AAEV,UAAM,YAAY,IAAI,OAAO;AAC7B,cAAU,kBAAkB,KAAK,cAAc;AAE/C,UAAM,eAAe,MAAM;AACzB,gBAAU,kBAAkB;AAC5B,gBAAU,MAAM;AAChB,aAAO,oBAAoB,SAAS,YAAY;AAAA,IAClD;AACA,WAAO,iBAAiB,SAAS,YAAY;AAE7C,QAAI;AACF,uBAAiB,MAAM,WAAW;AAChC,YAAI,OAAO,QAAS;AAEpB,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,OAAO,MAAM,2BAA2B;AAC7C,iBAAK,MAAM,gBAAgB,EAAE;AAC7B,iBAAK,WAAW;AAEhB,uBAAK,kBAAL,mBAAoB;AACpB;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,MAAM,mBAAmB,EAAE;AAChC;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,OAAO,MAAM,yBAAyB;AAC3C,iBAAK,MAAM,cAAc,EAAE;AAC3B,iBAAK,WAAW;AAEhB,iBAAK,mBAAmB,KAAK,IAAI,IAAI,GAAG;AAExC,gBACE,KAAK,wBACJ,KAAK,sBAAsB,SAAS,KAAK,mBAC1C;AACA,oBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,mBAAK,gBAAgB,OAAO;AAAA,YAC9B;AACA;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,GAAG,mBAAmB;AAAA,IAC1C,UAAE;AACA,WAAK,OAAO,MAAM,iBAAiB;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,oBAAoB,aAAyC;AAC3D,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,yBAAyB;AACvB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA,EAEA,gBAAgB;AAtZlB;AAuZI,SAAK,kBAAkB;AACvB,SAAK,yBAAyB;AAC9B,SAAK,oBAAoB;AAEzB,eAAK,YAAL,mBAAc,gBAAgB,QAAQ,MAAM;AAC1C,WAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,WAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,aAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,MACpD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,eAAe,eAAwB;AAnazC;AAoaI,UAAM,qBACJ,CAAC,gBAAwB,QACzB,OAAO,eAAgC;AACrC,UAAI,KAAK,IAAI,IAAI,KAAK,0BAA0B,eAAe;AAE7D,YAAI,iBAAiB,KAAK,eAAe,QAAW;AAClD,gBAAM,aAAa,KAAK,MAAM,KAAK,aAAa,GAAG;AACnD,gBAAM,UAAU,IAAI,WAAW,aAAa,CAAC;AAC7C,gBAAM,eAAe,IAAI,WAAW,SAAS,KAAK,YAAY,GAAG,UAAU;AAC3E,eAAK,mBAAmB,MAAM,YAAY;AAAA,QAC5C;AAGA,cAAM,MAAM,eAAe,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,MAC1D;AAEA,UAAI,KAAK,wBAAwB;AAE/B,aAAK,kBAAkB,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,MACvF;AACA,WAAK,yBAAyB;AAE9B,YAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,WAAK,OAAO,MAAM,yCAAyC;AAC3D,WAAK,gBAAgB,OAAO;AAC5B,WAAK,oBAAoB;AAAA,IAC3B;AAGF,eAAK,uBAAL,mBAAyB;AACzB,SAAK,qBAAqB,KAAK,KAAK,mBAAmB,CAAC;AAExD,SAAK,mBAAmB,OACrB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,qBAAqB;AAAA,IACzC,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,WAAK,OAAO,MAAM,KAAK,iCAAiC;AAAA,IAC1D,CAAC;AAAA,EACL;AAAA,EAEA,MAAM,QAAQ;AA7chB;AA8cI,SAAK,uBAAuB;AAC5B,YAAM,UAAK,uBAAL,mBAAyB;AAC/B,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,kBAAL,mBAAoB;AAAA,EAC5B;AAAA,EAEA,IAAY,uBAAuB;AACjC,WAAO,CAAC,OAAO,MAAS,EAAE,SAAS,KAAK,iBAAiB;AAAA,EAC3D;AACF;","names":[]}

package/dist/voice/events.cjs

@@ -0,0 +1,145 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var events_exports = {};
+__export(events_exports, {
+  AgentSessionEventTypes: () => AgentSessionEventTypes,
+  CloseReason: () => CloseReason,
+  createAgentStateChangedEvent: () => createAgentStateChangedEvent,
+  createCloseEvent: () => createCloseEvent,
+  createConversationItemAddedEvent: () => createConversationItemAddedEvent,
+  createErrorEvent: () => createErrorEvent,
+  createFunctionToolsExecutedEvent: () => createFunctionToolsExecutedEvent,
+  createMetricsCollectedEvent: () => createMetricsCollectedEvent,
+  createSpeechCreatedEvent: () => createSpeechCreatedEvent,
+  createUserInputTranscribedEvent: () => createUserInputTranscribedEvent,
+  createUserStateChangedEvent: () => createUserStateChangedEvent,
+  zipFunctionCallsAndOutputs: () => zipFunctionCallsAndOutputs
+});
+module.exports = __toCommonJS(events_exports);
+var AgentSessionEventTypes = /* @__PURE__ */ ((AgentSessionEventTypes2) => {
+  AgentSessionEventTypes2["UserInputTranscribed"] = "user_input_transcribed";
+  AgentSessionEventTypes2["AgentStateChanged"] = "agent_state_changed";
+  AgentSessionEventTypes2["UserStateChanged"] = "user_state_changed";
+  AgentSessionEventTypes2["ConversationItemAdded"] = "conversation_item_added";
+  AgentSessionEventTypes2["FunctionToolsExecuted"] = "function_tools_executed";
+  AgentSessionEventTypes2["MetricsCollected"] = "metrics_collected";
+  AgentSessionEventTypes2["SpeechCreated"] = "speech_created";
+  AgentSessionEventTypes2["Error"] = "error";
+  AgentSessionEventTypes2["Close"] = "close";
+  return AgentSessionEventTypes2;
+})(AgentSessionEventTypes || {});
+var CloseReason = /* @__PURE__ */ ((CloseReason2) => {
+  CloseReason2["ERROR"] = "error";
+  CloseReason2["JOB_SHUTDOWN"] = "job_shutdown";
+  CloseReason2["PARTICIPANT_DISCONNECTED"] = "participant_disconnected";
+  CloseReason2["USER_INITIATED"] = "user_initiated";
+  return CloseReason2;
+})(CloseReason || {});
+const createUserStateChangedEvent = (oldState, newState, createdAt = Date.now()) => ({
+  type: "user_state_changed",
+  oldState,
+  newState,
+  createdAt
+});
+const createAgentStateChangedEvent = (oldState, newState, createdAt = Date.now()) => ({
+  type: "agent_state_changed",
+  oldState,
+  newState,
+  createdAt
+});
+const createUserInputTranscribedEvent = ({
+  transcript,
+  isFinal,
+  speakerId = null,
+  createdAt = Date.now()
+}) => ({
+  type: "user_input_transcribed",
+  transcript,
+  isFinal,
+  speakerId,
+  createdAt
+});
+const createMetricsCollectedEvent = ({
+  metrics,
+  createdAt = Date.now()
+}) => ({
+  type: "metrics_collected",
+  metrics,
+  createdAt
+});
+const createConversationItemAddedEvent = (item, createdAt = Date.now()) => ({
+  type: "conversation_item_added",
+  item,
+  createdAt
+});
+const createFunctionToolsExecutedEvent = ({
+  functionCalls,
+  functionCallOutputs,
+  createdAt = Date.now()
+}) => {
+  return {
+    type: "function_tools_executed",
+    functionCalls,
+    functionCallOutputs,
+    createdAt
+  };
+};
+const zipFunctionCallsAndOutputs = (event) => {
+  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]]);
+};
+const createSpeechCreatedEvent = ({
+  userInitiated,
+  source,
+  speechHandle,
+  createdAt = Date.now()
+}) => ({
+  type: "speech_created",
+  userInitiated,
+  source,
+  speechHandle,
+  createdAt
+});
+const createErrorEvent = (error, source, createdAt = Date.now()) => ({
+  type: "error",
+  error,
+  source,
+  createdAt
+});
+const createCloseEvent = (reason, error = null, createdAt = Date.now()) => ({
+  type: "close",
+  error,
+  reason,
+  createdAt
+});
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AgentSessionEventTypes,
+  CloseReason,
+  createAgentStateChangedEvent,
+  createCloseEvent,
+  createConversationItemAddedEvent,
+  createErrorEvent,
+  createFunctionToolsExecutedEvent,
+  createMetricsCollectedEvent,
+  createSpeechCreatedEvent,
+  createUserInputTranscribedEvent,
+  createUserStateChangedEvent,
+  zipFunctionCallsAndOutputs
+});
+//# sourceMappingURL=events.cjs.map

package/dist/voice/events.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLM, RealtimeModel } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  speakerId: string | null;\n  createdAt: number;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | null;\n  reason: CloseReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: CloseReason,\n  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AATE,SAAAA;AAAA,GAAA;AAeL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAgBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAUO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,YAAY,KAAK,IAAI;AACvB,OAKkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAAoE,MACpE,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}