@livekit/agents 1.0.14 → 1.0.16

package/dist/voice/audio_recognition.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"audio_recognition.d.ts","sourceRoot":"","sources":["../../src/voice/audio_recognition.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAK1D,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,eAAe,CAAC;AAElE,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,QAAQ,EAAgB,MAAM,WAAW,CAAC;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEvC,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC3C,aAAa,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACtC,mBAAmB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC/C,iBAAiB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7C,WAAW,EAAE,CAAC,IAAI,EAAE,aAAa,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAEvD,eAAe,EAAE,MAAM,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC5B,iBAAiB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACtE,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,gBAAgB,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,iBAAiB,CAAC,EAAE,OAAO,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;IAC9D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,GAAG,CAAC,CAAU;IACtB,OAAO,CAAC,GAAG,CAAC,CAAM;IAClB,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAS;IAE9B,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,sBAAsB,CAAM;IACpC,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAC,CAAS;IAE5B,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,qBAAqB,CAAuC;IACpE,OAAO,CAAC,kBAAkB,CAA0C;IAGpE,OAAO,CAAC,aAAa,CAAC,CAAa;IACnC,OAAO,CAAC,kBAAkB,CAAC,CAAa;IACxC,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,OAAO,CAAC,CAAa;gBAEjB,IAAI,EAAE,uBAAuB;IAiBzC;;OAEG;IACH,IAAI,iBAAiB,IAAI,MAAM,CAK9B;IAEK,KAAK;YAYG,UAAU;IAiFxB,OAAO,CAAC,eAAe;YA4FT,aAAa;YAqDb,aAAa;IAyD3B,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAI3D,sBAAsB;IAItB,aAAa;IAab,cAAc,CAAC,aAAa,EAAE,OAAO;IA0C/B,KAAK;IAQX,OAAO,KAAK,oBAAoB,GAE/B;CACF"}
+{"version":3,"file":"audio_recognition.d.ts","sourceRoot":"","sources":["../../src/voice/audio_recognition.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAK1D,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,eAAe,CAAC;AAElE,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,QAAQ,EAAgB,MAAM,WAAW,CAAC;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEvC,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC3C,aAAa,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACtC,mBAAmB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC/C,iBAAiB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7C,WAAW,EAAE,CAAC,IAAI,EAAE,aAAa,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,sBAAsB,EAAE,CAAC,IAAI,EAAE,wBAAwB,KAAK,IAAI,CAAC;IAEjE,eAAe,EAAE,MAAM,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC5B,iBAAiB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACtE,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,gBAAgB,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,iBAAiB,CAAC,EAAE,OAAO,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;IAC9D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,GAAG,CAAC,CAAU;IACtB,OAAO,CAAC,GAAG,CAAC,CAAM;IAClB,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAS;IAE9B,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,sBAAsB,CAAM;IACpC,OAAO,CAAC,wBAAwB,CAAM;IACtC,OAAO,CAAC,yBAAyB,CAAgB;IACjD,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,eAAe,CAAqB;IAC5C,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAC,CAAS;IAE5B,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,qBAAqB,CAAuC;IACpE,OAAO,CAAC,kBAAkB,CAA0C;IAGpE,OAAO,CAAC,aAAa,CAAC,CAAa;IACnC,OAAO,CAAC,kBAAkB,CAAC,CAAa;IACxC,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,OAAO,CAAC,CAAa;gBAEjB,IAAI,EAAE,uBAAuB;IAiBzC;;OAEG;IACH,IAAI,iBAAiB,IAAI,MAAM,CAK9B;IAEK,KAAK;YAYG,UAAU;IAgMxB,OAAO,CAAC,eAAe;YAyIT,aAAa;YAqDb,aAAa;IAiE3B,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAI3D,sBAAsB;IAItB,aAAa;IAeb,cAAc,CAAC,aAAa,EAAE,OAAO;IA0C/B,KAAK;IAQX,OAAO,KAAK,oBAAoB,GAE/B;CACF"}

package/dist/voice/audio_recognition.js

@@ -22,7 +22,10 @@ class AudioRecognition {
   lastFinalTranscriptTime = 0;
   audioTranscript = "";
   audioInterimTranscript = "";
-  lastSpeakingTime = 0;
+  audioPreflightTranscript = "";
+  finalTranscriptConfidence = [];
+  lastSpeakingTime;
+  speechStartTime;
   userTurnCommitted = false;
   speaking = false;
   sampleRate;
@@ -70,7 +73,7 @@ class AudioRecognition {
     });
   }
   async onSTTEvent(ev) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r;
     if (this.turnDetectionMode === "manual" && this.userTurnCommitted && (this.bounceEOUTask === void 0 || this.bounceEOUTask.done || ev.type == SpeechEventType.INTERIM_TRANSCRIPT)) {
       this.logger.debug(
         {
@@ -87,7 +90,8 @@ class AudioRecognition {
       case SpeechEventType.FINAL_TRANSCRIPT:
         this.hooks.onFinalTranscript(ev);
         const transcript = (_c = (_b = ev.alternatives) == null ? void 0 : _b[0]) == null ? void 0 : _c.text;
-        this.lastLanguage = (_e = (_d = ev.alternatives) == null ? void 0 : _d[0]) == null ? void 0 : _e.language;
+        const confidence = ((_e = (_d = ev.alternatives) == null ? void 0 : _d[0]) == null ? void 0 : _e.confidence) ?? 0;
+        this.lastLanguage = (_g = (_f = ev.alternatives) == null ? void 0 : _f[0]) == null ? void 0 : _g.language;
         if (!transcript) {
           return;
         }
@@ -101,26 +105,112 @@ class AudioRecognition {
         this.lastFinalTranscriptTime = Date.now();
         this.audioTranscript += ` ${transcript}`;
         this.audioTranscript = this.audioTranscript.trimStart();
+        this.finalTranscriptConfidence.push(confidence);
+        const transcriptChanged = this.audioTranscript !== this.audioPreflightTranscript;
         this.audioInterimTranscript = "";
-        if (!this.speaking) {
-          if (!this.vad) {
-            this.lastSpeakingTime = Date.now();
+        this.audioPreflightTranscript = "";
+        if (!this.vad || this.lastSpeakingTime === void 0) {
+          this.lastSpeakingTime = Date.now();
+        }
+        if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (transcriptChanged) {
+            this.logger.debug(
+              { transcript: this.audioTranscript },
+              "triggering preemptive generation (FINAL_TRANSCRIPT)"
+            );
+            this.hooks.onPreemptiveGeneration({
+              newTranscript: this.audioTranscript,
+              transcriptConfidence: this.finalTranscriptConfidence.length > 0 ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) / this.finalTranscriptConfidence.length : 0
+            });
           }
-          if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (!this.speaking) {
             const chatCtx = this.hooks.retrieveChatCtx();
             this.logger.debug("running EOU detection on stt FINAL_TRANSCRIPT");
             this.runEOUDetection(chatCtx);
           }
         }
         break;
+      case SpeechEventType.PREFLIGHT_TRANSCRIPT:
+        this.hooks.onInterimTranscript(ev);
+        const preflightTranscript = ((_i = (_h = ev.alternatives) == null ? void 0 : _h[0]) == null ? void 0 : _i.text) ?? "";
+        const preflightConfidence = ((_k = (_j = ev.alternatives) == null ? void 0 : _j[0]) == null ? void 0 : _k.confidence) ?? 0;
+        const preflightLanguage = (_m = (_l = ev.alternatives) == null ? void 0 : _l[0]) == null ? void 0 : _m.language;
+        const MIN_LANGUAGE_DETECTION_LENGTH = 5;
+        if (!this.lastLanguage || preflightLanguage && preflightTranscript.length > MIN_LANGUAGE_DETECTION_LENGTH) {
+          this.lastLanguage = preflightLanguage;
+        }
+        if (!preflightTranscript) {
+          return;
+        }
+        this.logger.debug(
+          {
+            user_transcript: preflightTranscript,
+            language: this.lastLanguage
+          },
+          "received user preflight transcript"
+        );
+        this.lastFinalTranscriptTime = Date.now();
+        this.audioPreflightTranscript = `${this.audioTranscript} ${preflightTranscript}`.trimStart();
+        this.audioInterimTranscript = preflightTranscript;
+        if (!this.vad || this.lastSpeakingTime === void 0) {
+          this.lastSpeakingTime = Date.now();
+        }
+        if (this.turnDetectionMode !== "manual" || this.userTurnCommitted) {
+          const confidenceVals = [...this.finalTranscriptConfidence, preflightConfidence];
+          this.logger.debug(
+            {
+              transcript: this.audioPreflightTranscript.length > 100 ? this.audioPreflightTranscript.slice(0, 100) + "..." : this.audioPreflightTranscript
+            },
+            "triggering preemptive generation (PREFLIGHT_TRANSCRIPT)"
+          );
+          this.hooks.onPreemptiveGeneration({
+            newTranscript: this.audioPreflightTranscript,
+            transcriptConfidence: confidenceVals.length > 0 ? confidenceVals.reduce((a, b) => a + b, 0) / confidenceVals.length : 0
+          });
+        }
+        break;
       case SpeechEventType.INTERIM_TRANSCRIPT:
-        this.logger.debug({ transcript: (_g = (_f = ev.alternatives) == null ? void 0 : _f[0]) == null ? void 0 : _g.text }, "interim transcript");
+        this.logger.debug({ transcript: (_o = (_n = ev.alternatives) == null ? void 0 : _n[0]) == null ? void 0 : _o.text }, "interim transcript");
         this.hooks.onInterimTranscript(ev);
-        this.audioInterimTranscript = ((_i = (_h = ev.alternatives) == null ? void 0 : _h[0]) == null ? void 0 : _i.text) ?? "";
+        this.audioInterimTranscript = ((_q = (_p = ev.alternatives) == null ? void 0 : _p[0]) == null ? void 0 : _q.text) ?? "";
+        break;
+      case SpeechEventType.START_OF_SPEECH:
+        if (this.turnDetectionMode !== "stt") break;
+        this.hooks.onStartOfSpeech({
+          type: VADEventType.START_OF_SPEECH,
+          samplesIndex: 0,
+          timestamp: Date.now(),
+          speechDuration: 0,
+          silenceDuration: 0,
+          frames: [],
+          probability: 0,
+          inferenceDuration: 0,
+          speaking: true,
+          rawAccumulatedSilence: 0,
+          rawAccumulatedSpeech: 0
+        });
+        this.speaking = true;
+        this.lastSpeakingTime = Date.now();
+        (_r = this.bounceEOUTask) == null ? void 0 : _r.cancel();
         break;
       case SpeechEventType.END_OF_SPEECH:
         if (this.turnDetectionMode !== "stt") break;
+        this.hooks.onEndOfSpeech({
+          type: VADEventType.END_OF_SPEECH,
+          samplesIndex: 0,
+          timestamp: Date.now(),
+          speechDuration: 0,
+          silenceDuration: 0,
+          frames: [],
+          probability: 0,
+          inferenceDuration: 0,
+          speaking: false,
+          rawAccumulatedSilence: 0,
+          rawAccumulatedSpeech: 0
+        });
+        this.speaking = false;
         this.userTurnCommitted = true;
+        this.lastSpeakingTime = Date.now();
         if (!this.speaking) {
           const chatCtx = this.hooks.retrieveChatCtx();
           this.logger.debug("running EOU detection on stt END_OF_SPEECH");
@@ -148,7 +238,7 @@ class AudioRecognition {
       // disable EOU model if manual turn detection enabled
       this.audioTranscript && this.turnDetectionMode !== "manual" ? this.turnDetector : void 0
     );
-    const bounceEOUTask = (lastSpeakingTime) => async (controller) => {
+    const bounceEOUTask = (lastSpeakingTime, lastFinalTranscriptTime, speechStartTime) => async (controller) => {
       let endpointingDelay = this.minEndpointingDelay;
       if (turnDetector) {
         this.logger.debug("Running turn detector model");
@@ -175,21 +265,46 @@ class AudioRecognition {
           }
         }
       }
-      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();
-      await delay(Math.max(extraSleep, 0), { signal: controller.signal });
+      let extraSleep = endpointingDelay;
+      if (lastSpeakingTime !== void 0) {
+        extraSleep += lastSpeakingTime - Date.now();
+      }
+      if (extraSleep > 0) {
+        await delay(Math.max(extraSleep, 0), { signal: controller.signal });
+      }
       this.logger.debug({ transcript: this.audioTranscript }, "end of user turn");
+      const confidenceAvg = this.finalTranscriptConfidence.length > 0 ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) / this.finalTranscriptConfidence.length : 0;
+      let startedSpeakingAt;
+      let stoppedSpeakingAt;
+      let transcriptionDelay;
+      let endOfUtteranceDelay;
+      if (lastFinalTranscriptTime !== 0 && lastSpeakingTime !== void 0 && speechStartTime !== void 0) {
+        startedSpeakingAt = speechStartTime;
+        stoppedSpeakingAt = lastSpeakingTime;
+        transcriptionDelay = Math.max(lastFinalTranscriptTime - lastSpeakingTime, 0);
+        endOfUtteranceDelay = Date.now() - lastSpeakingTime;
+      }
       const committed = await this.hooks.onEndOfTurn({
         newTranscript: this.audioTranscript,
-        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),
-        endOfUtteranceDelay: Date.now() - lastSpeakingTime
+        transcriptConfidence: confidenceAvg,
+        transcriptionDelay: transcriptionDelay ?? 0,
+        endOfUtteranceDelay: endOfUtteranceDelay ?? 0,
+        startedSpeakingAt,
+        stoppedSpeakingAt
       });
       if (committed) {
         this.audioTranscript = "";
+        this.finalTranscriptConfidence = [];
+        this.lastSpeakingTime = void 0;
+        this.lastFinalTranscriptTime = 0;
+        this.speechStartTime = void 0;
       }
       this.userTurnCommitted = false;
     };
     (_a = this.bounceEOUTask) == null ? void 0 : _a.cancel();
-    this.bounceEOUTask = Task.from(bounceEOUTask(this.lastSpeakingTime));
+    this.bounceEOUTask = Task.from(
+      bounceEOUTask(this.lastSpeakingTime, this.lastFinalTranscriptTime, this.speechStartTime)
+    );
     this.bounceEOUTask.result.then(() => {
       this.logger.debug("EOU detection task completed");
     }).catch((err) => {
@@ -269,12 +384,17 @@ class AudioRecognition {
             break;
           case VADEventType.INFERENCE_DONE:
             this.hooks.onVADInferenceDone(ev);
+            if (ev.rawAccumulatedSpeech > 0) {
+              this.lastSpeakingTime = Date.now();
+              if (this.speechStartTime === void 0) {
+                this.speechStartTime = Date.now();
+              }
+            }
             break;
           case VADEventType.END_OF_SPEECH:
             this.logger.debug("VAD task: END_OF_SPEECH");
             this.hooks.onEndOfSpeech(ev);
             this.speaking = false;
-            this.lastSpeakingTime = Date.now() - ev.silenceDuration;
             if (this.vadBaseTurnDetection || this.turnDetectionMode === "stt" && this.userTurnCommitted) {
               const chatCtx = this.hooks.retrieveChatCtx();
               this.runEOUDetection(chatCtx);
@@ -298,6 +418,8 @@ class AudioRecognition {
     var _a;
     this.audioTranscript = "";
     this.audioInterimTranscript = "";
+    this.audioPreflightTranscript = "";
+    this.finalTranscriptConfidence = [];
     this.userTurnCommitted = false;
     (_a = this.sttTask) == null ? void 0 : _a.cancelAndWait().finally(() => {
       this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));

package/dist/voice/audio_recognition.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/audio_recognition.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { WritableStreamDefaultWriter } from 'node:stream/web';\nimport { ReadableStream } from 'node:stream/web';\nimport { type ChatContext } from '../llm/chat_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream, isStreamReaderReleaseError } from '../stream/deferred_stream.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { mergeReadableStreams } from '../stream/merge_readable_streams.js';\nimport { type SpeechEvent, SpeechEventType } from '../stt/stt.js';\nimport { Task, delay } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { TurnDetectionMode } from './agent_session.js';\nimport type { STTNode } from './io.js';\n\nexport interface EndOfTurnInfo {\n  newTranscript: string;\n  transcriptionDelay: number;\n  endOfUtteranceDelay: number;\n}\n\nexport interface RecognitionHooks {\n  onStartOfSpeech: (ev: VADEvent) => void;\n  onVADInferenceDone: (ev: VADEvent) => void;\n  onEndOfSpeech: (ev: VADEvent) => void;\n  onInterimTranscript: (ev: SpeechEvent) => void;\n  onFinalTranscript: (ev: SpeechEvent) => void;\n  onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;\n\n  retrieveChatCtx: () => ChatContext;\n}\n\nexport interface _TurnDetector {\n  unlikelyThreshold: (language?: string) => Promise<number | undefined>;\n  supportsLanguage: (language?: string) => Promise<boolean>;\n  predictEndOfTurn(chatCtx: ChatContext): Promise<number>;\n}\n\nexport interface AudioRecognitionOptions {\n  recognitionHooks: RecognitionHooks;\n  stt?: STTNode;\n  vad?: VAD;\n  turnDetector?: _TurnDetector;\n  turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n}\n\nexport class AudioRecognition {\n  private hooks: RecognitionHooks;\n  private stt?: STTNode;\n  private vad?: VAD;\n  private turnDetector?: _TurnDetector;\n  private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  private minEndpointingDelay: number;\n  private maxEndpointingDelay: number;\n  private lastLanguage?: string;\n\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private lastFinalTranscriptTime = 0;\n  private audioTranscript = '';\n  private audioInterimTranscript = '';\n  private lastSpeakingTime = 0;\n  private userTurnCommitted = false;\n  private speaking = false;\n  private sampleRate?: number;\n\n  private vadInputStream: ReadableStream<AudioFrame>;\n  private sttInputStream: ReadableStream<AudioFrame>;\n  private silenceAudioTransform = new IdentityTransform<AudioFrame>();\n  private silenceAudioWriter: WritableStreamDefaultWriter<AudioFrame>;\n\n  // all cancellable tasks\n  private bounceEOUTask?: Task<void>;\n  private commitUserTurnTask?: Task<void>;\n  private vadTask?: Task<void>;\n  private sttTask?: Task<void>;\n\n  constructor(opts: AudioRecognitionOptions) {\n    this.hooks = opts.recognitionHooks;\n    this.stt = opts.stt;\n    this.vad = opts.vad;\n    this.turnDetector = opts.turnDetector;\n    this.turnDetectionMode = opts.turnDetectionMode;\n    this.minEndpointingDelay = opts.minEndpointingDelay;\n    this.maxEndpointingDelay = opts.maxEndpointingDelay;\n    this.lastLanguage = undefined;\n\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();\n    this.vadInputStream = vadInputStream;\n    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);\n    this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();\n  }\n\n  /**\n   * Current transcript of the user's speech, including interim transcript if available.\n   */\n  get currentTranscript(): string {\n    if (this.audioInterimTranscript) {\n      return `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n    }\n    return this.audioTranscript;\n  }\n\n  async start() {\n    this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));\n    this.vadTask.result.catch((err) => {\n      this.logger.error(`Error running VAD task: ${err}`);\n    });\n\n    this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n    this.sttTask.result.catch((err) => {\n      this.logger.error(`Error running STT task: ${err}`);\n    });\n  }\n\n  private async onSTTEvent(ev: SpeechEvent) {\n    if (\n      this.turnDetectionMode === 'manual' &&\n      this.userTurnCommitted &&\n      (this.bounceEOUTask === undefined ||\n        this.bounceEOUTask.done ||\n        ev.type == SpeechEventType.INTERIM_TRANSCRIPT)\n    ) {\n      // ignore stt event if user turn already committed and EOU task is done\n      // or it's an interim transcript\n      this.logger.debug(\n        {\n          userTurnCommitted: this.userTurnCommitted,\n          eouTaskDone: this.bounceEOUTask?.done,\n          evType: ev.type,\n          turnDetectionMode: this.turnDetectionMode,\n        },\n        'ignoring stt event',\n      );\n      return;\n    }\n\n    switch (ev.type) {\n      case SpeechEventType.FINAL_TRANSCRIPT:\n        this.hooks.onFinalTranscript(ev);\n        const transcript = ev.alternatives?.[0]?.text;\n        this.lastLanguage = ev.alternatives?.[0]?.language;\n\n        if (!transcript) {\n          // stt final transcript received but no transcript\n          return;\n        }\n\n        this.logger.debug(\n          {\n            user_transcript: transcript,\n            language: this.lastLanguage,\n          },\n          'received user transcript',\n        );\n\n        this.lastFinalTranscriptTime = Date.now();\n        this.audioTranscript += ` ${transcript}`;\n        this.audioTranscript = this.audioTranscript.trimStart();\n        this.audioInterimTranscript = '';\n\n        if (!this.speaking) {\n          if (!this.vad) {\n            // Copied from python agents:\n            // vad disabled, use stt timestamp\n            // TODO: this would screw up transcription latency metrics\n            // but we'll live with it for now.\n            // the correct way is to ensure STT fires SpeechEventType.END_OF_SPEECH\n            // and using that timestamp for _last_speaking_time\n            this.lastSpeakingTime = Date.now();\n          }\n\n          if (this.vadBaseTurnDetection || this.userTurnCommitted) {\n            const chatCtx = this.hooks.retrieveChatCtx();\n            this.logger.debug('running EOU detection on stt FINAL_TRANSCRIPT');\n            this.runEOUDetection(chatCtx);\n          }\n        }\n        break;\n      case SpeechEventType.INTERIM_TRANSCRIPT:\n        this.logger.debug({ transcript: ev.alternatives?.[0]?.text }, 'interim transcript');\n        this.hooks.onInterimTranscript(ev);\n        this.audioInterimTranscript = ev.alternatives?.[0]?.text ?? '';\n        break;\n      case SpeechEventType.END_OF_SPEECH:\n        if (this.turnDetectionMode !== 'stt') break;\n        this.userTurnCommitted = true;\n\n        if (!this.speaking) {\n          const chatCtx = this.hooks.retrieveChatCtx();\n          this.logger.debug('running EOU detection on stt END_OF_SPEECH');\n          this.runEOUDetection(chatCtx);\n        }\n    }\n  }\n\n  private runEOUDetection(chatCtx: ChatContext) {\n    this.logger.debug(\n      {\n        stt: this.stt,\n        audioTranscript: this.audioTranscript,\n        turnDetectionMode: this.turnDetectionMode,\n      },\n      'running EOU detection',\n    );\n\n    if (this.stt && !this.audioTranscript && this.turnDetectionMode !== 'manual') {\n      // stt enabled but no transcript yet\n      this.logger.debug('skipping EOU detection');\n      return;\n    }\n\n    chatCtx = chatCtx.copy();\n    chatCtx.addMessage({ role: 'user', content: this.audioTranscript });\n\n    const turnDetector =\n      // disable EOU model if manual turn detection enabled\n      this.audioTranscript && this.turnDetectionMode !== 'manual' ? this.turnDetector : undefined;\n\n    const bounceEOUTask = (lastSpeakingTime: number) => async (controller: AbortController) => {\n      let endpointingDelay = this.minEndpointingDelay;\n\n      // TODO(AJS-74): need to support actual turn detection model plugins for following code to run\n      if (turnDetector) {\n        this.logger.debug('Running turn detector model');\n        if (!turnDetector.supportsLanguage(this.lastLanguage)) {\n          this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);\n        } else {\n          const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);\n          this.logger.debug(\n            { endOfTurnProbability, language: this.lastLanguage },\n            'end of turn probability',\n          );\n\n          const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);\n          this.logger.debug(\n            {\n              unlikelyThreshold,\n              endOfTurnProbability,\n              language: this.lastLanguage,\n              transcript: this.audioTranscript,\n            },\n            'EOU Detection',\n          );\n\n          if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {\n            endpointingDelay = this.maxEndpointingDelay;\n          }\n        }\n      }\n\n      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();\n      // add delay to see if there's a potential upcoming EOU task that cancels this one\n      await delay(Math.max(extraSleep, 0), { signal: controller.signal });\n\n      this.logger.debug({ transcript: this.audioTranscript }, 'end of user turn');\n\n      const committed = await this.hooks.onEndOfTurn({\n        newTranscript: this.audioTranscript,\n        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),\n        endOfUtteranceDelay: Date.now() - lastSpeakingTime,\n      });\n\n      if (committed) {\n        // clear the transcript if the user turn was committed\n        this.audioTranscript = '';\n      }\n\n      this.userTurnCommitted = false;\n    };\n\n    // cancel any existing EOU task\n    this.bounceEOUTask?.cancel();\n    this.bounceEOUTask = Task.from(bounceEOUTask(this.lastSpeakingTime));\n\n    this.bounceEOUTask.result\n      .then(() => {\n        this.logger.debug('EOU detection task completed');\n      })\n      .catch((err: unknown) => {\n        if (err instanceof Error && err.message.includes('This operation was aborted')) {\n          // ignore aborted errors\n          return;\n        }\n        this.logger.error(err, 'Error in EOU detection task:');\n      });\n  }\n\n  private async createSttTask(stt: STTNode | undefined, signal: AbortSignal) {\n    if (!stt) return;\n\n    this.logger.debug('createSttTask: create stt stream from stt node');\n\n    const sttStream = await stt(this.sttInputStream, {});\n\n    if (signal.aborted || sttStream === null) return;\n\n    if (sttStream instanceof ReadableStream) {\n      const reader = sttStream.getReader();\n\n      signal.addEventListener('abort', async () => {\n        try {\n          reader.releaseLock();\n          await sttStream?.cancel();\n        } catch (e) {\n          this.logger.debug('createSttTask: error during abort handler:', e);\n        }\n      });\n\n      try {\n        while (true) {\n          if (signal.aborted) break;\n\n          const { done, value: ev } = await reader.read();\n          if (done) break;\n\n          if (typeof ev === 'string') {\n            throw new Error('STT node must yield SpeechEvent');\n          } else {\n            await this.onSTTEvent(ev);\n          }\n        }\n      } catch (e) {\n        if (isStreamReaderReleaseError(e)) {\n          return;\n        }\n        this.logger.error({ error: e }, 'createSttTask: error reading sttStream');\n      } finally {\n        reader.releaseLock();\n        try {\n          await sttStream.cancel();\n        } catch (e) {\n          this.logger.debug(\n            'createSttTask: error cancelling sttStream (may already be cancelled):',\n            e,\n          );\n        }\n      }\n    }\n  }\n\n  private async createVadTask(vad: VAD | undefined, signal: AbortSignal) {\n    if (!vad) return;\n\n    const vadStream = vad.stream();\n    vadStream.updateInputStream(this.vadInputStream);\n\n    const abortHandler = () => {\n      vadStream.detachInputStream();\n      vadStream.close();\n      signal.removeEventListener('abort', abortHandler);\n    };\n    signal.addEventListener('abort', abortHandler);\n\n    try {\n      for await (const ev of vadStream) {\n        if (signal.aborted) break;\n\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.logger.debug('VAD task: START_OF_SPEECH');\n            this.hooks.onStartOfSpeech(ev);\n            this.speaking = true;\n\n            // Capture sample rate from the first VAD event if not already set\n            if (ev.frames.length > 0 && ev.frames[0]) {\n              this.sampleRate = ev.frames[0].sampleRate;\n            }\n\n            this.bounceEOUTask?.cancel();\n            break;\n          case VADEventType.INFERENCE_DONE:\n            this.hooks.onVADInferenceDone(ev);\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.logger.debug('VAD task: END_OF_SPEECH');\n            this.hooks.onEndOfSpeech(ev);\n            this.speaking = false;\n            // when VAD fires END_OF_SPEECH, it already waited for the silence_duration\n            this.lastSpeakingTime = Date.now() - ev.silenceDuration;\n\n            if (\n              this.vadBaseTurnDetection ||\n              (this.turnDetectionMode === 'stt' && this.userTurnCommitted)\n            ) {\n              const chatCtx = this.hooks.retrieveChatCtx();\n              this.runEOUDetection(chatCtx);\n            }\n            break;\n        }\n      }\n    } catch (e) {\n      this.logger.error(e, 'Error in VAD task');\n    } finally {\n      this.logger.debug('VAD task closed');\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputAudioStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  clearUserTurn() {\n    this.audioTranscript = '';\n    this.audioInterimTranscript = '';\n    this.userTurnCommitted = false;\n\n    this.sttTask?.cancelAndWait().finally(() => {\n      this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n      this.sttTask.result.catch((err) => {\n        this.logger.error(`Error running STT task: ${err}`);\n      });\n    });\n  }\n\n  commitUserTurn(audioDetached: boolean) {\n    const commitUserTurnTask =\n      (delayDuration: number = 500) =>\n      async (controller: AbortController) => {\n        if (Date.now() - this.lastFinalTranscriptTime > delayDuration) {\n          // flush the stt by pushing silence\n          if (audioDetached && this.sampleRate !== undefined) {\n            const numSamples = Math.floor(this.sampleRate * 0.5);\n            const silence = new Int16Array(numSamples * 2);\n            const silenceFrame = new AudioFrame(silence, this.sampleRate, 1, numSamples);\n            this.silenceAudioWriter.write(silenceFrame);\n          }\n\n          // wait for the final transcript to be available\n          await delay(delayDuration, { signal: controller.signal });\n        }\n\n        if (this.audioInterimTranscript) {\n          // append interim transcript in case the final transcript is not ready\n          this.audioTranscript = `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n        }\n        this.audioInterimTranscript = '';\n\n        const chatCtx = this.hooks.retrieveChatCtx();\n        this.logger.debug('running EOU detection on commitUserTurn');\n        this.runEOUDetection(chatCtx);\n        this.userTurnCommitted = true;\n      };\n\n    // cancel any existing commit user turn task\n    this.commitUserTurnTask?.cancel();\n    this.commitUserTurnTask = Task.from(commitUserTurnTask());\n\n    this.commitUserTurnTask.result\n      .then(() => {\n        this.logger.debug('User turn committed');\n      })\n      .catch((err: unknown) => {\n        this.logger.error(err, 'Error in user turn commit task:');\n      });\n  }\n\n  async close() {\n    this.detachInputAudioStream();\n    await this.commitUserTurnTask?.cancelAndWait();\n    await this.sttTask?.cancelAndWait();\n    await this.vadTask?.cancelAndWait();\n    await this.bounceEOUTask?.cancelAndWait();\n  }\n\n  private get vadBaseTurnDetection() {\n    return ['vad', undefined].includes(this.turnDetectionMode);\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAE3B,SAAS,sBAAsB;AAC/B,eAAiC;AACjC,SAAS,WAAW;AACpB,SAAS,wBAAwB,kCAAkC;AACnE,SAAS,yBAAyB;AAClC,SAAS,4BAA4B;AACrC,SAA2B,uBAAuB;AAClD,SAAS,MAAM,aAAa;AAC5B,SAAkC,oBAAoB;AAqC/C,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA,SAAS,IAAI;AAAA,EACb,0BAA0B;AAAA,EAC1B,kBAAkB;AAAA,EAClB,yBAAyB;AAAA,EACzB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,WAAW;AAAA,EACX;AAAA,EAEA;AAAA,EACA;AAAA,EACA,wBAAwB,IAAI,kBAA8B;AAAA,EAC1D;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,MAA+B;AACzC,SAAK,QAAQ,KAAK;AAClB,SAAK,MAAM,KAAK;AAChB,SAAK,MAAM,KAAK;AAChB,SAAK,eAAe,KAAK;AACzB,SAAK,oBAAoB,KAAK;AAC9B,SAAK,sBAAsB,KAAK;AAChC,SAAK,sBAAsB,KAAK;AAChC,SAAK,eAAe;AAEpB,SAAK,sBAAsB,IAAI,uBAAmC;AAClE,UAAM,CAAC,gBAAgB,cAAc,IAAI,KAAK,oBAAoB,OAAO,IAAI;AAC7E,SAAK,iBAAiB;AACtB,SAAK,iBAAiB,qBAAqB,gBAAgB,KAAK,sBAAsB,QAAQ;AAC9F,SAAK,qBAAqB,KAAK,sBAAsB,SAAS,UAAU;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,oBAA4B;AAC9B,QAAI,KAAK,wBAAwB;AAC/B,aAAO,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,IACvE;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAED,SAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,WAAW,IAAiB;AAxH5C;AAyHI,QACE,KAAK,sBAAsB,YAC3B,KAAK,sBACJ,KAAK,kBAAkB,UACtB,KAAK,cAAc,QACnB,GAAG,QAAQ,gBAAgB,qBAC7B;AAGA,WAAK,OAAO;AAAA,QACV;AAAA,UACE,mBAAmB,KAAK;AAAA,UACxB,cAAa,UAAK,kBAAL,mBAAoB;AAAA,UACjC,QAAQ,GAAG;AAAA,UACX,mBAAmB,KAAK;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AACA;AAAA,IACF;AAEA,YAAQ,GAAG,MAAM;AAAA,MACf,KAAK,gBAAgB;AACnB,aAAK,MAAM,kBAAkB,EAAE;AAC/B,cAAM,cAAa,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AACzC,aAAK,gBAAe,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AAE1C,YAAI,CAAC,YAAY;AAEf;AAAA,QACF;AAEA,aAAK,OAAO;AAAA,UACV;AAAA,YACE,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,UACjB;AAAA,UACA;AAAA,QACF;AAEA,aAAK,0BAA0B,KAAK,IAAI;AACxC,aAAK,mBAAmB,IAAI,UAAU;AACtC,aAAK,kBAAkB,KAAK,gBAAgB,UAAU;AACtD,aAAK,yBAAyB;AAE9B,YAAI,CAAC,KAAK,UAAU;AAClB,cAAI,CAAC,KAAK,KAAK;AAOb,iBAAK,mBAAmB,KAAK,IAAI;AAAA,UACnC;AAEA,cAAI,KAAK,wBAAwB,KAAK,mBAAmB;AACvD,kBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,iBAAK,OAAO,MAAM,+CAA+C;AACjE,iBAAK,gBAAgB,OAAO;AAAA,UAC9B;AAAA,QACF;AACA;AAAA,MACF,KAAK,gBAAgB;AACnB,aAAK,OAAO,MAAM,EAAE,aAAY,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,KAAK,GAAG,oBAAoB;AAClF,aAAK,MAAM,oBAAoB,EAAE;AACjC,aAAK,2BAAyB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,SAAQ;AAC5D;AAAA,MACF,KAAK,gBAAgB;AACnB,YAAI,KAAK,sBAAsB,MAAO;AACtC,aAAK,oBAAoB;AAEzB,YAAI,CAAC,KAAK,UAAU;AAClB,gBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,eAAK,OAAO,MAAM,4CAA4C;AAC9D,eAAK,gBAAgB,OAAO;AAAA,QAC9B;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,gBAAgB,SAAsB;AAzMhD;AA0MI,SAAK,OAAO;AAAA,MACV;AAAA,QACE,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,mBAAmB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,CAAC,KAAK,mBAAmB,KAAK,sBAAsB,UAAU;AAE5E,WAAK,OAAO,MAAM,wBAAwB;AAC1C;AAAA,IACF;AAEA,cAAU,QAAQ,KAAK;AACvB,YAAQ,WAAW,EAAE,MAAM,QAAQ,SAAS,KAAK,gBAAgB,CAAC;AAElE,UAAM;AAAA;AAAA,MAEJ,KAAK,mBAAmB,KAAK,sBAAsB,WAAW,KAAK,eAAe;AAAA;AAEpF,UAAM,gBAAgB,CAAC,qBAA6B,OAAO,eAAgC;AACzF,UAAI,mBAAmB,KAAK;AAG5B,UAAI,cAAc;AAChB,aAAK,OAAO,MAAM,6BAA6B;AAC/C,YAAI,CAAC,aAAa,iBAAiB,KAAK,YAAY,GAAG;AACrD,eAAK,OAAO,MAAM,2CAA2C,KAAK,YAAY,EAAE;AAAA,QAClF,OAAO;AACL,gBAAM,uBAAuB,MAAM,aAAa,iBAAiB,OAAO;AACxE,eAAK,OAAO;AAAA,YACV,EAAE,sBAAsB,UAAU,KAAK,aAAa;AAAA,YACpD;AAAA,UACF;AAEA,gBAAM,oBAAoB,MAAM,aAAa,kBAAkB,KAAK,YAAY;AAChF,eAAK,OAAO;AAAA,YACV;AAAA,cACE;AAAA,cACA;AAAA,cACA,UAAU,KAAK;AAAA,cACf,YAAY,KAAK;AAAA,YACnB;AAAA,YACA;AAAA,UACF;AAEA,cAAI,qBAAqB,uBAAuB,mBAAmB;AACjE,+BAAmB,KAAK;AAAA,UAC1B;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,mBAAmB,mBAAmB,KAAK,IAAI;AAElE,YAAM,MAAM,KAAK,IAAI,YAAY,CAAC,GAAG,EAAE,QAAQ,WAAW,OAAO,CAAC;AAElE,WAAK,OAAO,MAAM,EAAE,YAAY,KAAK,gBAAgB,GAAG,kBAAkB;AAE1E,YAAM,YAAY,MAAM,KAAK,MAAM,YAAY;AAAA,QAC7C,eAAe,KAAK;AAAA,QACpB,oBAAoB,KAAK,IAAI,KAAK,0BAA0B,kBAAkB,CAAC;AAAA,QAC/E,qBAAqB,KAAK,IAAI,IAAI;AAAA,MACpC,CAAC;AAED,UAAI,WAAW;AAEb,aAAK,kBAAkB;AAAA,MACzB;AAEA,WAAK,oBAAoB;AAAA,IAC3B;AAGA,eAAK,kBAAL,mBAAoB;AACpB,SAAK,gBAAgB,KAAK,KAAK,cAAc,KAAK,gBAAgB,CAAC;AAEnE,SAAK,cAAc,OAChB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,8BAA8B;AAAA,IAClD,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,UAAI,eAAe,SAAS,IAAI,QAAQ,SAAS,4BAA4B,GAAG;AAE9E;AAAA,MACF;AACA,WAAK,OAAO,MAAM,KAAK,8BAA8B;AAAA,IACvD,CAAC;AAAA,EACL;AAAA,EAEA,MAAc,cAAc,KAA0B,QAAqB;AACzE,QAAI,CAAC,IAAK;AAEV,SAAK,OAAO,MAAM,gDAAgD;AAElE,UAAM,YAAY,MAAM,IAAI,KAAK,gBAAgB,CAAC,CAAC;AAEnD,QAAI,OAAO,WAAW,cAAc,KAAM;AAE1C,QAAI,qBAAqB,gBAAgB;AACvC,YAAM,SAAS,UAAU,UAAU;AAEnC,aAAO,iBAAiB,SAAS,YAAY;AAC3C,YAAI;AACF,iBAAO,YAAY;AACnB,iBAAM,uCAAW;AAAA,QACnB,SAAS,GAAG;AACV,eAAK,OAAO,MAAM,8CAA8C,CAAC;AAAA,QACnE;AAAA,MACF,CAAC;AAED,UAAI;AACF,eAAO,MAAM;AACX,cAAI,OAAO,QAAS;AAEpB,gBAAM,EAAE,MAAM,OAAO,GAAG,IAAI,MAAM,OAAO,KAAK;AAC9C,cAAI,KAAM;AAEV,cAAI,OAAO,OAAO,UAAU;AAC1B,kBAAM,IAAI,MAAM,iCAAiC;AAAA,UACnD,OAAO;AACL,kBAAM,KAAK,WAAW,EAAE;AAAA,UAC1B;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,YAAI,2BAA2B,CAAC,GAAG;AACjC;AAAA,QACF;AACA,aAAK,OAAO,MAAM,EAAE,OAAO,EAAE,GAAG,wCAAwC;AAAA,MAC1E,UAAE;AACA,eAAO,YAAY;AACnB,YAAI;AACF,gBAAM,UAAU,OAAO;AAAA,QACzB,SAAS,GAAG;AACV,eAAK,OAAO;AAAA,YACV;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,KAAsB,QAAqB;AA1VzE;AA2VI,QAAI,CAAC,IAAK;AAEV,UAAM,YAAY,IAAI,OAAO;AAC7B,cAAU,kBAAkB,KAAK,cAAc;AAE/C,UAAM,eAAe,MAAM;AACzB,gBAAU,kBAAkB;AAC5B,gBAAU,MAAM;AAChB,aAAO,oBAAoB,SAAS,YAAY;AAAA,IAClD;AACA,WAAO,iBAAiB,SAAS,YAAY;AAE7C,QAAI;AACF,uBAAiB,MAAM,WAAW;AAChC,YAAI,OAAO,QAAS;AAEpB,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,OAAO,MAAM,2BAA2B;AAC7C,iBAAK,MAAM,gBAAgB,EAAE;AAC7B,iBAAK,WAAW;AAGhB,gBAAI,GAAG,OAAO,SAAS,KAAK,GAAG,OAAO,CAAC,GAAG;AACxC,mBAAK,aAAa,GAAG,OAAO,CAAC,EAAE;AAAA,YACjC;AAEA,uBAAK,kBAAL,mBAAoB;AACpB;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,MAAM,mBAAmB,EAAE;AAChC;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,OAAO,MAAM,yBAAyB;AAC3C,iBAAK,MAAM,cAAc,EAAE;AAC3B,iBAAK,WAAW;AAEhB,iBAAK,mBAAmB,KAAK,IAAI,IAAI,GAAG;AAExC,gBACE,KAAK,wBACJ,KAAK,sBAAsB,SAAS,KAAK,mBAC1C;AACA,oBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,mBAAK,gBAAgB,OAAO;AAAA,YAC9B;AACA;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,GAAG,mBAAmB;AAAA,IAC1C,UAAE;AACA,WAAK,OAAO,MAAM,iBAAiB;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,oBAAoB,aAAyC;AAC3D,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,yBAAyB;AACvB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA,EAEA,gBAAgB;AA3ZlB;AA4ZI,SAAK,kBAAkB;AACvB,SAAK,yBAAyB;AAC9B,SAAK,oBAAoB;AAEzB,eAAK,YAAL,mBAAc,gBAAgB,QAAQ,MAAM;AAC1C,WAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,WAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,aAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,MACpD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,eAAe,eAAwB;AAxazC;AAyaI,UAAM,qBACJ,CAAC,gBAAwB,QACzB,OAAO,eAAgC;AACrC,UAAI,KAAK,IAAI,IAAI,KAAK,0BAA0B,eAAe;AAE7D,YAAI,iBAAiB,KAAK,eAAe,QAAW;AAClD,gBAAM,aAAa,KAAK,MAAM,KAAK,aAAa,GAAG;AACnD,gBAAM,UAAU,IAAI,WAAW,aAAa,CAAC;AAC7C,gBAAM,eAAe,IAAI,WAAW,SAAS,KAAK,YAAY,GAAG,UAAU;AAC3E,eAAK,mBAAmB,MAAM,YAAY;AAAA,QAC5C;AAGA,cAAM,MAAM,eAAe,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,MAC1D;AAEA,UAAI,KAAK,wBAAwB;AAE/B,aAAK,kBAAkB,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,MACvF;AACA,WAAK,yBAAyB;AAE9B,YAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,WAAK,OAAO,MAAM,yCAAyC;AAC3D,WAAK,gBAAgB,OAAO;AAC5B,WAAK,oBAAoB;AAAA,IAC3B;AAGF,eAAK,uBAAL,mBAAyB;AACzB,SAAK,qBAAqB,KAAK,KAAK,mBAAmB,CAAC;AAExD,SAAK,mBAAmB,OACrB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,qBAAqB;AAAA,IACzC,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,WAAK,OAAO,MAAM,KAAK,iCAAiC;AAAA,IAC1D,CAAC;AAAA,EACL;AAAA,EAEA,MAAM,QAAQ;AAldhB;AAmdI,SAAK,uBAAuB;AAC5B,YAAM,UAAK,uBAAL,mBAAyB;AAC/B,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,kBAAL,mBAAoB;AAAA,EAC5B;AAAA,EAEA,IAAY,uBAAuB;AACjC,WAAO,CAAC,OAAO,MAAS,EAAE,SAAS,KAAK,iBAAiB;AAAA,EAC3D;AACF;","names":[]}
+{"version":3,"sources":["../../src/voice/audio_recognition.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { WritableStreamDefaultWriter } from 'node:stream/web';\nimport { ReadableStream } from 'node:stream/web';\nimport { type ChatContext } from '../llm/chat_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream, isStreamReaderReleaseError } from '../stream/deferred_stream.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { mergeReadableStreams } from '../stream/merge_readable_streams.js';\nimport { type SpeechEvent, SpeechEventType } from '../stt/stt.js';\nimport { Task, delay } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { TurnDetectionMode } from './agent_session.js';\nimport type { STTNode } from './io.js';\n\nexport interface EndOfTurnInfo {\n  newTranscript: string;\n  transcriptConfidence: number;\n  transcriptionDelay: number;\n  endOfUtteranceDelay: number;\n  startedSpeakingAt: number | undefined;\n  stoppedSpeakingAt: number | undefined;\n}\n\nexport interface PreemptiveGenerationInfo {\n  newTranscript: string;\n  transcriptConfidence: number;\n}\n\nexport interface RecognitionHooks {\n  onStartOfSpeech: (ev: VADEvent) => void;\n  onVADInferenceDone: (ev: VADEvent) => void;\n  onEndOfSpeech: (ev: VADEvent) => void;\n  onInterimTranscript: (ev: SpeechEvent) => void;\n  onFinalTranscript: (ev: SpeechEvent) => void;\n  onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;\n  onPreemptiveGeneration: (info: PreemptiveGenerationInfo) => void;\n\n  retrieveChatCtx: () => ChatContext;\n}\n\nexport interface _TurnDetector {\n  unlikelyThreshold: (language?: string) => Promise<number | undefined>;\n  supportsLanguage: (language?: string) => Promise<boolean>;\n  predictEndOfTurn(chatCtx: ChatContext): Promise<number>;\n}\n\nexport interface AudioRecognitionOptions {\n  recognitionHooks: RecognitionHooks;\n  stt?: STTNode;\n  vad?: VAD;\n  turnDetector?: _TurnDetector;\n  turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n}\n\nexport class AudioRecognition {\n  private hooks: RecognitionHooks;\n  private stt?: STTNode;\n  private vad?: VAD;\n  private turnDetector?: _TurnDetector;\n  private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  private minEndpointingDelay: number;\n  private maxEndpointingDelay: number;\n  private lastLanguage?: string;\n\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private lastFinalTranscriptTime = 0;\n  private audioTranscript = '';\n  private audioInterimTranscript = '';\n  private audioPreflightTranscript = '';\n  private finalTranscriptConfidence: number[] = [];\n  private lastSpeakingTime: number | undefined;\n  private speechStartTime: number | undefined;\n  private userTurnCommitted = false;\n  private speaking = false;\n  private sampleRate?: number;\n\n  private vadInputStream: ReadableStream<AudioFrame>;\n  private sttInputStream: ReadableStream<AudioFrame>;\n  private silenceAudioTransform = new IdentityTransform<AudioFrame>();\n  private silenceAudioWriter: WritableStreamDefaultWriter<AudioFrame>;\n\n  // all cancellable tasks\n  private bounceEOUTask?: Task<void>;\n  private commitUserTurnTask?: Task<void>;\n  private vadTask?: Task<void>;\n  private sttTask?: Task<void>;\n\n  constructor(opts: AudioRecognitionOptions) {\n    this.hooks = opts.recognitionHooks;\n    this.stt = opts.stt;\n    this.vad = opts.vad;\n    this.turnDetector = opts.turnDetector;\n    this.turnDetectionMode = opts.turnDetectionMode;\n    this.minEndpointingDelay = opts.minEndpointingDelay;\n    this.maxEndpointingDelay = opts.maxEndpointingDelay;\n    this.lastLanguage = undefined;\n\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();\n    this.vadInputStream = vadInputStream;\n    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);\n    this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();\n  }\n\n  /**\n   * Current transcript of the user's speech, including interim transcript if available.\n   */\n  get currentTranscript(): string {\n    if (this.audioInterimTranscript) {\n      return `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n    }\n    return this.audioTranscript;\n  }\n\n  async start() {\n    this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));\n    this.vadTask.result.catch((err) => {\n      this.logger.error(`Error running VAD task: ${err}`);\n    });\n\n    this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n    this.sttTask.result.catch((err) => {\n      this.logger.error(`Error running STT task: ${err}`);\n    });\n  }\n\n  private async onSTTEvent(ev: SpeechEvent) {\n    if (\n      this.turnDetectionMode === 'manual' &&\n      this.userTurnCommitted &&\n      (this.bounceEOUTask === undefined ||\n        this.bounceEOUTask.done ||\n        ev.type == SpeechEventType.INTERIM_TRANSCRIPT)\n    ) {\n      // ignore stt event if user turn already committed and EOU task is done\n      // or it's an interim transcript\n      this.logger.debug(\n        {\n          userTurnCommitted: this.userTurnCommitted,\n          eouTaskDone: this.bounceEOUTask?.done,\n          evType: ev.type,\n          turnDetectionMode: this.turnDetectionMode,\n        },\n        'ignoring stt event',\n      );\n      return;\n    }\n\n    switch (ev.type) {\n      case SpeechEventType.FINAL_TRANSCRIPT:\n        this.hooks.onFinalTranscript(ev);\n        const transcript = ev.alternatives?.[0]?.text;\n        const confidence = ev.alternatives?.[0]?.confidence ?? 0;\n        this.lastLanguage = ev.alternatives?.[0]?.language;\n\n        if (!transcript) {\n          // stt final transcript received but no transcript\n          return;\n        }\n\n        this.logger.debug(\n          {\n            user_transcript: transcript,\n            language: this.lastLanguage,\n          },\n          'received user transcript',\n        );\n\n        this.lastFinalTranscriptTime = Date.now();\n        this.audioTranscript += ` ${transcript}`;\n        this.audioTranscript = this.audioTranscript.trimStart();\n        this.finalTranscriptConfidence.push(confidence);\n        const transcriptChanged = this.audioTranscript !== this.audioPreflightTranscript;\n        this.audioInterimTranscript = '';\n        this.audioPreflightTranscript = '';\n\n        if (!this.vad || this.lastSpeakingTime === undefined) {\n          // vad disabled, use stt timestamp\n          // TODO: this would screw up transcription latency metrics\n          // but we'll live with it for now.\n          // the correct way is to ensure STT fires SpeechEventType.END_OF_SPEECH\n          // and using that timestamp for lastSpeakingTime\n          this.lastSpeakingTime = Date.now();\n        }\n\n        if (this.vadBaseTurnDetection || this.userTurnCommitted) {\n          if (transcriptChanged) {\n            this.logger.debug(\n              { transcript: this.audioTranscript },\n              'triggering preemptive generation (FINAL_TRANSCRIPT)',\n            );\n            this.hooks.onPreemptiveGeneration({\n              newTranscript: this.audioTranscript,\n              transcriptConfidence:\n                this.finalTranscriptConfidence.length > 0\n                  ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) /\n                    this.finalTranscriptConfidence.length\n                  : 0,\n            });\n          }\n\n          if (!this.speaking) {\n            const chatCtx = this.hooks.retrieveChatCtx();\n            this.logger.debug('running EOU detection on stt FINAL_TRANSCRIPT');\n            this.runEOUDetection(chatCtx);\n          }\n        }\n        break;\n      case SpeechEventType.PREFLIGHT_TRANSCRIPT:\n        this.hooks.onInterimTranscript(ev);\n        const preflightTranscript = ev.alternatives?.[0]?.text ?? '';\n        const preflightConfidence = ev.alternatives?.[0]?.confidence ?? 0;\n        const preflightLanguage = ev.alternatives?.[0]?.language;\n\n        const MIN_LANGUAGE_DETECTION_LENGTH = 5;\n        if (\n          !this.lastLanguage ||\n          (preflightLanguage && preflightTranscript.length > MIN_LANGUAGE_DETECTION_LENGTH)\n        ) {\n          this.lastLanguage = preflightLanguage;\n        }\n\n        if (!preflightTranscript) {\n          return;\n        }\n\n        this.logger.debug(\n          {\n            user_transcript: preflightTranscript,\n            language: this.lastLanguage,\n          },\n          'received user preflight transcript',\n        );\n\n        // still need to increment it as it's used for turn detection,\n        this.lastFinalTranscriptTime = Date.now();\n        // preflight transcript includes all pre-committed transcripts (including final transcript from the previous STT run)\n        this.audioPreflightTranscript =\n          `${this.audioTranscript} ${preflightTranscript}`.trimStart();\n        this.audioInterimTranscript = preflightTranscript;\n\n        if (!this.vad || this.lastSpeakingTime === undefined) {\n          // vad disabled, use stt timestamp\n          this.lastSpeakingTime = Date.now();\n        }\n\n        if (this.turnDetectionMode !== 'manual' || this.userTurnCommitted) {\n          const confidenceVals = [...this.finalTranscriptConfidence, preflightConfidence];\n          this.logger.debug(\n            {\n              transcript:\n                this.audioPreflightTranscript.length > 100\n                  ? this.audioPreflightTranscript.slice(0, 100) + '...'\n                  : this.audioPreflightTranscript,\n            },\n            'triggering preemptive generation (PREFLIGHT_TRANSCRIPT)',\n          );\n          this.hooks.onPreemptiveGeneration({\n            newTranscript: this.audioPreflightTranscript,\n            transcriptConfidence:\n              confidenceVals.length > 0\n                ? confidenceVals.reduce((a, b) => a + b, 0) / confidenceVals.length\n                : 0,\n          });\n        }\n        break;\n      case SpeechEventType.INTERIM_TRANSCRIPT:\n        this.logger.debug({ transcript: ev.alternatives?.[0]?.text }, 'interim transcript');\n        this.hooks.onInterimTranscript(ev);\n        this.audioInterimTranscript = ev.alternatives?.[0]?.text ?? '';\n        break;\n      case SpeechEventType.START_OF_SPEECH:\n        if (this.turnDetectionMode !== 'stt') break;\n        this.hooks.onStartOfSpeech({\n          type: VADEventType.START_OF_SPEECH,\n          samplesIndex: 0,\n          timestamp: Date.now(),\n          speechDuration: 0,\n          silenceDuration: 0,\n          frames: [],\n          probability: 0,\n          inferenceDuration: 0,\n          speaking: true,\n          rawAccumulatedSilence: 0,\n          rawAccumulatedSpeech: 0,\n        });\n        this.speaking = true;\n        this.lastSpeakingTime = Date.now();\n\n        this.bounceEOUTask?.cancel();\n        break;\n      case SpeechEventType.END_OF_SPEECH:\n        if (this.turnDetectionMode !== 'stt') break;\n        this.hooks.onEndOfSpeech({\n          type: VADEventType.END_OF_SPEECH,\n          samplesIndex: 0,\n          timestamp: Date.now(),\n          speechDuration: 0,\n          silenceDuration: 0,\n          frames: [],\n          probability: 0,\n          inferenceDuration: 0,\n          speaking: false,\n          rawAccumulatedSilence: 0,\n          rawAccumulatedSpeech: 0,\n        });\n        this.speaking = false;\n        this.userTurnCommitted = true;\n        this.lastSpeakingTime = Date.now();\n\n        if (!this.speaking) {\n          const chatCtx = this.hooks.retrieveChatCtx();\n          this.logger.debug('running EOU detection on stt END_OF_SPEECH');\n          this.runEOUDetection(chatCtx);\n        }\n    }\n  }\n\n  private runEOUDetection(chatCtx: ChatContext) {\n    this.logger.debug(\n      {\n        stt: this.stt,\n        audioTranscript: this.audioTranscript,\n        turnDetectionMode: this.turnDetectionMode,\n      },\n      'running EOU detection',\n    );\n\n    if (this.stt && !this.audioTranscript && this.turnDetectionMode !== 'manual') {\n      // stt enabled but no transcript yet\n      this.logger.debug('skipping EOU detection');\n      return;\n    }\n\n    chatCtx = chatCtx.copy();\n    chatCtx.addMessage({ role: 'user', content: this.audioTranscript });\n\n    const turnDetector =\n      // disable EOU model if manual turn detection enabled\n      this.audioTranscript && this.turnDetectionMode !== 'manual' ? this.turnDetector : undefined;\n\n    const bounceEOUTask =\n      (\n        lastSpeakingTime: number | undefined,\n        lastFinalTranscriptTime: number,\n        speechStartTime: number | undefined,\n      ) =>\n      async (controller: AbortController) => {\n        let endpointingDelay = this.minEndpointingDelay;\n\n        if (turnDetector) {\n          this.logger.debug('Running turn detector model');\n          if (!turnDetector.supportsLanguage(this.lastLanguage)) {\n            this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);\n          } else {\n            const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);\n            this.logger.debug(\n              { endOfTurnProbability, language: this.lastLanguage },\n              'end of turn probability',\n            );\n\n            const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);\n            this.logger.debug(\n              {\n                unlikelyThreshold,\n                endOfTurnProbability,\n                language: this.lastLanguage,\n                transcript: this.audioTranscript,\n              },\n              'EOU Detection',\n            );\n\n            if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {\n              endpointingDelay = this.maxEndpointingDelay;\n            }\n          }\n        }\n\n        let extraSleep = endpointingDelay;\n        if (lastSpeakingTime !== undefined) {\n          extraSleep += lastSpeakingTime - Date.now();\n        }\n\n        if (extraSleep > 0) {\n          // add delay to see if there's a potential upcoming EOU task that cancels this one\n          await delay(Math.max(extraSleep, 0), { signal: controller.signal });\n        }\n\n        this.logger.debug({ transcript: this.audioTranscript }, 'end of user turn');\n\n        const confidenceAvg =\n          this.finalTranscriptConfidence.length > 0\n            ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) /\n              this.finalTranscriptConfidence.length\n            : 0;\n\n        let startedSpeakingAt: number | undefined;\n        let stoppedSpeakingAt: number | undefined;\n        let transcriptionDelay: number | undefined;\n        let endOfUtteranceDelay: number | undefined;\n\n        // sometimes, we can't calculate the metrics because VAD was unreliable.\n        // in this case, we just ignore the calculation, it's better than providing likely wrong values\n        if (\n          lastFinalTranscriptTime !== 0 &&\n          lastSpeakingTime !== undefined &&\n          speechStartTime !== undefined\n        ) {\n          startedSpeakingAt = speechStartTime;\n          stoppedSpeakingAt = lastSpeakingTime;\n          transcriptionDelay = Math.max(lastFinalTranscriptTime - lastSpeakingTime, 0);\n          endOfUtteranceDelay = Date.now() - lastSpeakingTime;\n        }\n\n        const committed = await this.hooks.onEndOfTurn({\n          newTranscript: this.audioTranscript,\n          transcriptConfidence: confidenceAvg,\n          transcriptionDelay: transcriptionDelay ?? 0,\n          endOfUtteranceDelay: endOfUtteranceDelay ?? 0,\n          startedSpeakingAt,\n          stoppedSpeakingAt,\n        });\n\n        if (committed) {\n          // clear the transcript if the user turn was committed\n          this.audioTranscript = '';\n          this.finalTranscriptConfidence = [];\n          this.lastSpeakingTime = undefined;\n          this.lastFinalTranscriptTime = 0;\n          this.speechStartTime = undefined;\n        }\n\n        this.userTurnCommitted = false;\n      };\n\n    // cancel any existing EOU task\n    this.bounceEOUTask?.cancel();\n    // copy the values before awaiting (the values can change)\n    this.bounceEOUTask = Task.from(\n      bounceEOUTask(this.lastSpeakingTime, this.lastFinalTranscriptTime, this.speechStartTime),\n    );\n\n    this.bounceEOUTask.result\n      .then(() => {\n        this.logger.debug('EOU detection task completed');\n      })\n      .catch((err: unknown) => {\n        if (err instanceof Error && err.message.includes('This operation was aborted')) {\n          // ignore aborted errors\n          return;\n        }\n        this.logger.error(err, 'Error in EOU detection task:');\n      });\n  }\n\n  private async createSttTask(stt: STTNode | undefined, signal: AbortSignal) {\n    if (!stt) return;\n\n    this.logger.debug('createSttTask: create stt stream from stt node');\n\n    const sttStream = await stt(this.sttInputStream, {});\n\n    if (signal.aborted || sttStream === null) return;\n\n    if (sttStream instanceof ReadableStream) {\n      const reader = sttStream.getReader();\n\n      signal.addEventListener('abort', async () => {\n        try {\n          reader.releaseLock();\n          await sttStream?.cancel();\n        } catch (e) {\n          this.logger.debug('createSttTask: error during abort handler:', e);\n        }\n      });\n\n      try {\n        while (true) {\n          if (signal.aborted) break;\n\n          const { done, value: ev } = await reader.read();\n          if (done) break;\n\n          if (typeof ev === 'string') {\n            throw new Error('STT node must yield SpeechEvent');\n          } else {\n            await this.onSTTEvent(ev);\n          }\n        }\n      } catch (e) {\n        if (isStreamReaderReleaseError(e)) {\n          return;\n        }\n        this.logger.error({ error: e }, 'createSttTask: error reading sttStream');\n      } finally {\n        reader.releaseLock();\n        try {\n          await sttStream.cancel();\n        } catch (e) {\n          this.logger.debug(\n            'createSttTask: error cancelling sttStream (may already be cancelled):',\n            e,\n          );\n        }\n      }\n    }\n  }\n\n  private async createVadTask(vad: VAD | undefined, signal: AbortSignal) {\n    if (!vad) return;\n\n    const vadStream = vad.stream();\n    vadStream.updateInputStream(this.vadInputStream);\n\n    const abortHandler = () => {\n      vadStream.detachInputStream();\n      vadStream.close();\n      signal.removeEventListener('abort', abortHandler);\n    };\n    signal.addEventListener('abort', abortHandler);\n\n    try {\n      for await (const ev of vadStream) {\n        if (signal.aborted) break;\n\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.logger.debug('VAD task: START_OF_SPEECH');\n            this.hooks.onStartOfSpeech(ev);\n            this.speaking = true;\n\n            // Capture sample rate from the first VAD event if not already set\n            if (ev.frames.length > 0 && ev.frames[0]) {\n              this.sampleRate = ev.frames[0].sampleRate;\n            }\n\n            this.bounceEOUTask?.cancel();\n            break;\n          case VADEventType.INFERENCE_DONE:\n            this.hooks.onVADInferenceDone(ev);\n            // for metrics, get the \"earliest\" signal of speech as possible\n            if (ev.rawAccumulatedSpeech > 0.0) {\n              this.lastSpeakingTime = Date.now();\n\n              if (this.speechStartTime === undefined) {\n                this.speechStartTime = Date.now();\n              }\n            }\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.logger.debug('VAD task: END_OF_SPEECH');\n            this.hooks.onEndOfSpeech(ev);\n\n            // when VAD fires END_OF_SPEECH, it already waited for the silence_duration\n            this.speaking = false;\n\n            if (\n              this.vadBaseTurnDetection ||\n              (this.turnDetectionMode === 'stt' && this.userTurnCommitted)\n            ) {\n              const chatCtx = this.hooks.retrieveChatCtx();\n              this.runEOUDetection(chatCtx);\n            }\n            break;\n        }\n      }\n    } catch (e) {\n      this.logger.error(e, 'Error in VAD task');\n    } finally {\n      this.logger.debug('VAD task closed');\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputAudioStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  clearUserTurn() {\n    this.audioTranscript = '';\n    this.audioInterimTranscript = '';\n    this.audioPreflightTranscript = '';\n    this.finalTranscriptConfidence = [];\n    this.userTurnCommitted = false;\n\n    this.sttTask?.cancelAndWait().finally(() => {\n      this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n      this.sttTask.result.catch((err) => {\n        this.logger.error(`Error running STT task: ${err}`);\n      });\n    });\n  }\n\n  commitUserTurn(audioDetached: boolean) {\n    const commitUserTurnTask =\n      (delayDuration: number = 500) =>\n      async (controller: AbortController) => {\n        if (Date.now() - this.lastFinalTranscriptTime > delayDuration) {\n          // flush the stt by pushing silence\n          if (audioDetached && this.sampleRate !== undefined) {\n            const numSamples = Math.floor(this.sampleRate * 0.5);\n            const silence = new Int16Array(numSamples * 2);\n            const silenceFrame = new AudioFrame(silence, this.sampleRate, 1, numSamples);\n            this.silenceAudioWriter.write(silenceFrame);\n          }\n\n          // wait for the final transcript to be available\n          await delay(delayDuration, { signal: controller.signal });\n        }\n\n        if (this.audioInterimTranscript) {\n          // append interim transcript in case the final transcript is not ready\n          this.audioTranscript = `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n        }\n        this.audioInterimTranscript = '';\n\n        const chatCtx = this.hooks.retrieveChatCtx();\n        this.logger.debug('running EOU detection on commitUserTurn');\n        this.runEOUDetection(chatCtx);\n        this.userTurnCommitted = true;\n      };\n\n    // cancel any existing commit user turn task\n    this.commitUserTurnTask?.cancel();\n    this.commitUserTurnTask = Task.from(commitUserTurnTask());\n\n    this.commitUserTurnTask.result\n      .then(() => {\n        this.logger.debug('User turn committed');\n      })\n      .catch((err: unknown) => {\n        this.logger.error(err, 'Error in user turn commit task:');\n      });\n  }\n\n  async close() {\n    this.detachInputAudioStream();\n    await this.commitUserTurnTask?.cancelAndWait();\n    await this.sttTask?.cancelAndWait();\n    await this.vadTask?.cancelAndWait();\n    await this.bounceEOUTask?.cancelAndWait();\n  }\n\n  private get vadBaseTurnDetection() {\n    return ['vad', undefined].includes(this.turnDetectionMode);\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAE3B,SAAS,sBAAsB;AAC/B,eAAiC;AACjC,SAAS,WAAW;AACpB,SAAS,wBAAwB,kCAAkC;AACnE,SAAS,yBAAyB;AAClC,SAAS,4BAA4B;AACrC,SAA2B,uBAAuB;AAClD,SAAS,MAAM,aAAa;AAC5B,SAAkC,oBAAoB;AA8C/C,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA,SAAS,IAAI;AAAA,EACb,0BAA0B;AAAA,EAC1B,kBAAkB;AAAA,EAClB,yBAAyB;AAAA,EACzB,2BAA2B;AAAA,EAC3B,4BAAsC,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA,oBAAoB;AAAA,EACpB,WAAW;AAAA,EACX;AAAA,EAEA;AAAA,EACA;AAAA,EACA,wBAAwB,IAAI,kBAA8B;AAAA,EAC1D;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,MAA+B;AACzC,SAAK,QAAQ,KAAK;AAClB,SAAK,MAAM,KAAK;AAChB,SAAK,MAAM,KAAK;AAChB,SAAK,eAAe,KAAK;AACzB,SAAK,oBAAoB,KAAK;AAC9B,SAAK,sBAAsB,KAAK;AAChC,SAAK,sBAAsB,KAAK;AAChC,SAAK,eAAe;AAEpB,SAAK,sBAAsB,IAAI,uBAAmC;AAClE,UAAM,CAAC,gBAAgB,cAAc,IAAI,KAAK,oBAAoB,OAAO,IAAI;AAC7E,SAAK,iBAAiB;AACtB,SAAK,iBAAiB,qBAAqB,gBAAgB,KAAK,sBAAsB,QAAQ;AAC9F,SAAK,qBAAqB,KAAK,sBAAsB,SAAS,UAAU;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,oBAA4B;AAC9B,QAAI,KAAK,wBAAwB;AAC/B,aAAO,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,IACvE;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAED,SAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,WAAW,IAAiB;AApI5C;AAqII,QACE,KAAK,sBAAsB,YAC3B,KAAK,sBACJ,KAAK,kBAAkB,UACtB,KAAK,cAAc,QACnB,GAAG,QAAQ,gBAAgB,qBAC7B;AAGA,WAAK,OAAO;AAAA,QACV;AAAA,UACE,mBAAmB,KAAK;AAAA,UACxB,cAAa,UAAK,kBAAL,mBAAoB;AAAA,UACjC,QAAQ,GAAG;AAAA,UACX,mBAAmB,KAAK;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AACA;AAAA,IACF;AAEA,YAAQ,GAAG,MAAM;AAAA,MACf,KAAK,gBAAgB;AACnB,aAAK,MAAM,kBAAkB,EAAE;AAC/B,cAAM,cAAa,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AACzC,cAAM,eAAa,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,eAAc;AACvD,aAAK,gBAAe,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AAE1C,YAAI,CAAC,YAAY;AAEf;AAAA,QACF;AAEA,aAAK,OAAO;AAAA,UACV;AAAA,YACE,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,UACjB;AAAA,UACA;AAAA,QACF;AAEA,aAAK,0BAA0B,KAAK,IAAI;AACxC,aAAK,mBAAmB,IAAI,UAAU;AACtC,aAAK,kBAAkB,KAAK,gBAAgB,UAAU;AACtD,aAAK,0BAA0B,KAAK,UAAU;AAC9C,cAAM,oBAAoB,KAAK,oBAAoB,KAAK;AACxD,aAAK,yBAAyB;AAC9B,aAAK,2BAA2B;AAEhC,YAAI,CAAC,KAAK,OAAO,KAAK,qBAAqB,QAAW;AAMpD,eAAK,mBAAmB,KAAK,IAAI;AAAA,QACnC;AAEA,YAAI,KAAK,wBAAwB,KAAK,mBAAmB;AACvD,cAAI,mBAAmB;AACrB,iBAAK,OAAO;AAAA,cACV,EAAE,YAAY,KAAK,gBAAgB;AAAA,cACnC;AAAA,YACF;AACA,iBAAK,MAAM,uBAAuB;AAAA,cAChC,eAAe,KAAK;AAAA,cACpB,sBACE,KAAK,0BAA0B,SAAS,IACpC,KAAK,0BAA0B,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IACxD,KAAK,0BAA0B,SAC/B;AAAA,YACR,CAAC;AAAA,UACH;AAEA,cAAI,CAAC,KAAK,UAAU;AAClB,kBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,iBAAK,OAAO,MAAM,+CAA+C;AACjE,iBAAK,gBAAgB,OAAO;AAAA,UAC9B;AAAA,QACF;AACA;AAAA,MACF,KAAK,gBAAgB;AACnB,aAAK,MAAM,oBAAoB,EAAE;AACjC,cAAM,wBAAsB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,SAAQ;AAC1D,cAAM,wBAAsB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,eAAc;AAChE,cAAM,qBAAoB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AAEhD,cAAM,gCAAgC;AACtC,YACE,CAAC,KAAK,gBACL,qBAAqB,oBAAoB,SAAS,+BACnD;AACA,eAAK,eAAe;AAAA,QACtB;AAEA,YAAI,CAAC,qBAAqB;AACxB;AAAA,QACF;AAEA,aAAK,OAAO;AAAA,UACV;AAAA,YACE,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,UACjB;AAAA,UACA;AAAA,QACF;AAGA,aAAK,0BAA0B,KAAK,IAAI;AAExC,aAAK,2BACH,GAAG,KAAK,eAAe,IAAI,mBAAmB,GAAG,UAAU;AAC7D,aAAK,yBAAyB;AAE9B,YAAI,CAAC,KAAK,OAAO,KAAK,qBAAqB,QAAW;AAEpD,eAAK,mBAAmB,KAAK,IAAI;AAAA,QACnC;AAEA,YAAI,KAAK,sBAAsB,YAAY,KAAK,mBAAmB;AACjE,gBAAM,iBAAiB,CAAC,GAAG,KAAK,2BAA2B,mBAAmB;AAC9E,eAAK,OAAO;AAAA,YACV;AAAA,cACE,YACE,KAAK,yBAAyB,SAAS,MACnC,KAAK,yBAAyB,MAAM,GAAG,GAAG,IAAI,QAC9C,KAAK;AAAA,YACb;AAAA,YACA;AAAA,UACF;AACA,eAAK,MAAM,uBAAuB;AAAA,YAChC,eAAe,KAAK;AAAA,YACpB,sBACE,eAAe,SAAS,IACpB,eAAe,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,eAAe,SAC3D;AAAA,UACR,CAAC;AAAA,QACH;AACA;AAAA,MACF,KAAK,gBAAgB;AACnB,aAAK,OAAO,MAAM,EAAE,aAAY,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,KAAK,GAAG,oBAAoB;AAClF,aAAK,MAAM,oBAAoB,EAAE;AACjC,aAAK,2BAAyB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,SAAQ;AAC5D;AAAA,MACF,KAAK,gBAAgB;AACnB,YAAI,KAAK,sBAAsB,MAAO;AACtC,aAAK,MAAM,gBAAgB;AAAA,UACzB,MAAM,aAAa;AAAA,UACnB,cAAc;AAAA,UACd,WAAW,KAAK,IAAI;AAAA,UACpB,gBAAgB;AAAA,UAChB,iBAAiB;AAAA,UACjB,QAAQ,CAAC;AAAA,UACT,aAAa;AAAA,UACb,mBAAmB;AAAA,UACnB,UAAU;AAAA,UACV,uBAAuB;AAAA,UACvB,sBAAsB;AAAA,QACxB,CAAC;AACD,aAAK,WAAW;AAChB,aAAK,mBAAmB,KAAK,IAAI;AAEjC,mBAAK,kBAAL,mBAAoB;AACpB;AAAA,MACF,KAAK,gBAAgB;AACnB,YAAI,KAAK,sBAAsB,MAAO;AACtC,aAAK,MAAM,cAAc;AAAA,UACvB,MAAM,aAAa;AAAA,UACnB,cAAc;AAAA,UACd,WAAW,KAAK,IAAI;AAAA,UACpB,gBAAgB;AAAA,UAChB,iBAAiB;AAAA,UACjB,QAAQ,CAAC;AAAA,UACT,aAAa;AAAA,UACb,mBAAmB;AAAA,UACnB,UAAU;AAAA,UACV,uBAAuB;AAAA,UACvB,sBAAsB;AAAA,QACxB,CAAC;AACD,aAAK,WAAW;AAChB,aAAK,oBAAoB;AACzB,aAAK,mBAAmB,KAAK,IAAI;AAEjC,YAAI,CAAC,KAAK,UAAU;AAClB,gBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,eAAK,OAAO,MAAM,4CAA4C;AAC9D,eAAK,gBAAgB,OAAO;AAAA,QAC9B;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,gBAAgB,SAAsB;AApUhD;AAqUI,SAAK,OAAO;AAAA,MACV;AAAA,QACE,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,mBAAmB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,CAAC,KAAK,mBAAmB,KAAK,sBAAsB,UAAU;AAE5E,WAAK,OAAO,MAAM,wBAAwB;AAC1C;AAAA,IACF;AAEA,cAAU,QAAQ,KAAK;AACvB,YAAQ,WAAW,EAAE,MAAM,QAAQ,SAAS,KAAK,gBAAgB,CAAC;AAElE,UAAM;AAAA;AAAA,MAEJ,KAAK,mBAAmB,KAAK,sBAAsB,WAAW,KAAK,eAAe;AAAA;AAEpF,UAAM,gBACJ,CACE,kBACA,yBACA,oBAEF,OAAO,eAAgC;AACrC,UAAI,mBAAmB,KAAK;AAE5B,UAAI,cAAc;AAChB,aAAK,OAAO,MAAM,6BAA6B;AAC/C,YAAI,CAAC,aAAa,iBAAiB,KAAK,YAAY,GAAG;AACrD,eAAK,OAAO,MAAM,2CAA2C,KAAK,YAAY,EAAE;AAAA,QAClF,OAAO;AACL,gBAAM,uBAAuB,MAAM,aAAa,iBAAiB,OAAO;AACxE,eAAK,OAAO;AAAA,YACV,EAAE,sBAAsB,UAAU,KAAK,aAAa;AAAA,YACpD;AAAA,UACF;AAEA,gBAAM,oBAAoB,MAAM,aAAa,kBAAkB,KAAK,YAAY;AAChF,eAAK,OAAO;AAAA,YACV;AAAA,cACE;AAAA,cACA;AAAA,cACA,UAAU,KAAK;AAAA,cACf,YAAY,KAAK;AAAA,YACnB;AAAA,YACA;AAAA,UACF;AAEA,cAAI,qBAAqB,uBAAuB,mBAAmB;AACjE,+BAAmB,KAAK;AAAA,UAC1B;AAAA,QACF;AAAA,MACF;AAEA,UAAI,aAAa;AACjB,UAAI,qBAAqB,QAAW;AAClC,sBAAc,mBAAmB,KAAK,IAAI;AAAA,MAC5C;AAEA,UAAI,aAAa,GAAG;AAElB,cAAM,MAAM,KAAK,IAAI,YAAY,CAAC,GAAG,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,MACpE;AAEA,WAAK,OAAO,MAAM,EAAE,YAAY,KAAK,gBAAgB,GAAG,kBAAkB;AAE1E,YAAM,gBACJ,KAAK,0BAA0B,SAAS,IACpC,KAAK,0BAA0B,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IACxD,KAAK,0BAA0B,SAC/B;AAEN,UAAI;AACJ,UAAI;AACJ,UAAI;AACJ,UAAI;AAIJ,UACE,4BAA4B,KAC5B,qBAAqB,UACrB,oBAAoB,QACpB;AACA,4BAAoB;AACpB,4BAAoB;AACpB,6BAAqB,KAAK,IAAI,0BAA0B,kBAAkB,CAAC;AAC3E,8BAAsB,KAAK,IAAI,IAAI;AAAA,MACrC;AAEA,YAAM,YAAY,MAAM,KAAK,MAAM,YAAY;AAAA,QAC7C,eAAe,KAAK;AAAA,QACpB,sBAAsB;AAAA,QACtB,oBAAoB,sBAAsB;AAAA,QAC1C,qBAAqB,uBAAuB;AAAA,QAC5C;AAAA,QACA;AAAA,MACF,CAAC;AAED,UAAI,WAAW;AAEb,aAAK,kBAAkB;AACvB,aAAK,4BAA4B,CAAC;AAClC,aAAK,mBAAmB;AACxB,aAAK,0BAA0B;AAC/B,aAAK,kBAAkB;AAAA,MACzB;AAEA,WAAK,oBAAoB;AAAA,IAC3B;AAGF,eAAK,kBAAL,mBAAoB;AAEpB,SAAK,gBAAgB,KAAK;AAAA,MACxB,cAAc,KAAK,kBAAkB,KAAK,yBAAyB,KAAK,eAAe;AAAA,IACzF;AAEA,SAAK,cAAc,OAChB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,8BAA8B;AAAA,IAClD,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,UAAI,eAAe,SAAS,IAAI,QAAQ,SAAS,4BAA4B,GAAG;AAE9E;AAAA,MACF;AACA,WAAK,OAAO,MAAM,KAAK,8BAA8B;AAAA,IACvD,CAAC;AAAA,EACL;AAAA,EAEA,MAAc,cAAc,KAA0B,QAAqB;AACzE,QAAI,CAAC,IAAK;AAEV,SAAK,OAAO,MAAM,gDAAgD;AAElE,UAAM,YAAY,MAAM,IAAI,KAAK,gBAAgB,CAAC,CAAC;AAEnD,QAAI,OAAO,WAAW,cAAc,KAAM;AAE1C,QAAI,qBAAqB,gBAAgB;AACvC,YAAM,SAAS,UAAU,UAAU;AAEnC,aAAO,iBAAiB,SAAS,YAAY;AAC3C,YAAI;AACF,iBAAO,YAAY;AACnB,iBAAM,uCAAW;AAAA,QACnB,SAAS,GAAG;AACV,eAAK,OAAO,MAAM,8CAA8C,CAAC;AAAA,QACnE;AAAA,MACF,CAAC;AAED,UAAI;AACF,eAAO,MAAM;AACX,cAAI,OAAO,QAAS;AAEpB,gBAAM,EAAE,MAAM,OAAO,GAAG,IAAI,MAAM,OAAO,KAAK;AAC9C,cAAI,KAAM;AAEV,cAAI,OAAO,OAAO,UAAU;AAC1B,kBAAM,IAAI,MAAM,iCAAiC;AAAA,UACnD,OAAO;AACL,kBAAM,KAAK,WAAW,EAAE;AAAA,UAC1B;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,YAAI,2BAA2B,CAAC,GAAG;AACjC;AAAA,QACF;AACA,aAAK,OAAO,MAAM,EAAE,OAAO,EAAE,GAAG,wCAAwC;AAAA,MAC1E,UAAE;AACA,eAAO,YAAY;AACnB,YAAI;AACF,gBAAM,UAAU,OAAO;AAAA,QACzB,SAAS,GAAG;AACV,eAAK,OAAO;AAAA,YACV;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,KAAsB,QAAqB;AAlgBzE;AAmgBI,QAAI,CAAC,IAAK;AAEV,UAAM,YAAY,IAAI,OAAO;AAC7B,cAAU,kBAAkB,KAAK,cAAc;AAE/C,UAAM,eAAe,MAAM;AACzB,gBAAU,kBAAkB;AAC5B,gBAAU,MAAM;AAChB,aAAO,oBAAoB,SAAS,YAAY;AAAA,IAClD;AACA,WAAO,iBAAiB,SAAS,YAAY;AAE7C,QAAI;AACF,uBAAiB,MAAM,WAAW;AAChC,YAAI,OAAO,QAAS;AAEpB,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,OAAO,MAAM,2BAA2B;AAC7C,iBAAK,MAAM,gBAAgB,EAAE;AAC7B,iBAAK,WAAW;AAGhB,gBAAI,GAAG,OAAO,SAAS,KAAK,GAAG,OAAO,CAAC,GAAG;AACxC,mBAAK,aAAa,GAAG,OAAO,CAAC,EAAE;AAAA,YACjC;AAEA,uBAAK,kBAAL,mBAAoB;AACpB;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,MAAM,mBAAmB,EAAE;AAEhC,gBAAI,GAAG,uBAAuB,GAAK;AACjC,mBAAK,mBAAmB,KAAK,IAAI;AAEjC,kBAAI,KAAK,oBAAoB,QAAW;AACtC,qBAAK,kBAAkB,KAAK,IAAI;AAAA,cAClC;AAAA,YACF;AACA;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,OAAO,MAAM,yBAAyB;AAC3C,iBAAK,MAAM,cAAc,EAAE;AAG3B,iBAAK,WAAW;AAEhB,gBACE,KAAK,wBACJ,KAAK,sBAAsB,SAAS,KAAK,mBAC1C;AACA,oBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,mBAAK,gBAAgB,OAAO;AAAA,YAC9B;AACA;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,GAAG,mBAAmB;AAAA,IAC1C,UAAE;AACA,WAAK,OAAO,MAAM,iBAAiB;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,oBAAoB,aAAyC;AAC3D,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,yBAAyB;AACvB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA,EAEA,gBAAgB;AA3kBlB;AA4kBI,SAAK,kBAAkB;AACvB,SAAK,yBAAyB;AAC9B,SAAK,2BAA2B;AAChC,SAAK,4BAA4B,CAAC;AAClC,SAAK,oBAAoB;AAEzB,eAAK,YAAL,mBAAc,gBAAgB,QAAQ,MAAM;AAC1C,WAAK,UAAU,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,WAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,aAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,MACpD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,eAAe,eAAwB;AA1lBzC;AA2lBI,UAAM,qBACJ,CAAC,gBAAwB,QACzB,OAAO,eAAgC;AACrC,UAAI,KAAK,IAAI,IAAI,KAAK,0BAA0B,eAAe;AAE7D,YAAI,iBAAiB,KAAK,eAAe,QAAW;AAClD,gBAAM,aAAa,KAAK,MAAM,KAAK,aAAa,GAAG;AACnD,gBAAM,UAAU,IAAI,WAAW,aAAa,CAAC;AAC7C,gBAAM,eAAe,IAAI,WAAW,SAAS,KAAK,YAAY,GAAG,UAAU;AAC3E,eAAK,mBAAmB,MAAM,YAAY;AAAA,QAC5C;AAGA,cAAM,MAAM,eAAe,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,MAC1D;AAEA,UAAI,KAAK,wBAAwB;AAE/B,aAAK,kBAAkB,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,MACvF;AACA,WAAK,yBAAyB;AAE9B,YAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,WAAK,OAAO,MAAM,yCAAyC;AAC3D,WAAK,gBAAgB,OAAO;AAC5B,WAAK,oBAAoB;AAAA,IAC3B;AAGF,eAAK,uBAAL,mBAAyB;AACzB,SAAK,qBAAqB,KAAK,KAAK,mBAAmB,CAAC;AAExD,SAAK,mBAAmB,OACrB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,qBAAqB;AAAA,IACzC,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,WAAK,OAAO,MAAM,KAAK,iCAAiC;AAAA,IAC1D,CAAC;AAAA,EACL;AAAA,EAEA,MAAM,QAAQ;AApoBhB;AAqoBI,SAAK,uBAAuB;AAC5B,YAAM,UAAK,uBAAL,mBAAyB;AAC/B,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,kBAAL,mBAAoB;AAAA,EAC5B;AAAA,EAEA,IAAY,uBAAuB;AACjC,WAAO,CAAC,OAAO,MAAS,EAAE,SAAS,KAAK,iBAAiB;AAAA,EAC3D;AACF;","names":[]}

package/dist/voice/generation.cjs

@@ -34,9 +34,9 @@ __export(generation_exports, {
 });
 module.exports = __toCommonJS(generation_exports);
 var import_rtc_node = require("@livekit/rtc-node");
-var import_zod = require("zod");
 var import_chat_context = require("../llm/chat_context.cjs");
 var import_tool_context = require("../llm/tool_context.cjs");
+var import_zod_utils = require("../llm/zod-utils.cjs");
 var import_log = require("../log.cjs");
 var import_identity_transform = require("../stream/identity_transform.cjs");
 var import_utils = require("../utils.cjs");
@@ -566,8 +566,13 @@ function performToolExecutions({
       let parsedArgs;
       try {
         const jsonArgs = JSON.parse(toolCall.args);
-        if (tool.parameters instanceof import_zod.ZodObject) {
-          parsedArgs = tool.parameters.parse(jsonArgs);
+        if ((0, import_zod_utils.isZodSchema)(tool.parameters)) {
+          const result = await (0, import_zod_utils.parseZodSchema)(tool.parameters, jsonArgs);
+          if (result.success) {
+            parsedArgs = result.data;
+          } else {
+            throw result.error;
+          }
         } else {
           parsedArgs = jsonArgs;
         }

package/dist/voice/generation.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/generation.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AudioResampler } from '@livekit/rtc-node';\nimport type { ReadableStream, ReadableStreamDefaultReader } from 'stream/web';\nimport { ZodObject } from 'zod';\nimport {\n  type ChatContext,\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n} from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport {\n  type ToolChoice,\n  type ToolContext,\n  isAgentHandoff,\n  isFunctionTool,\n  isToolError,\n} from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { Future, Task, shortuuid, toError } from '../utils.js';\nimport { type Agent, type ModelSettings, asyncLocalStorage, isStopResponse } from './agent.js';\nimport type { AgentSession } from './agent_session.js';\nimport type { AudioOutput, LLMNode, TTSNode, TextOutput } from './io.js';\nimport { RunContext } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\n/** @internal */\nexport class _LLMGenerationData {\n  generatedText: string = '';\n  generatedToolCalls: FunctionCall[];\n  id: string;\n\n  constructor(\n    public readonly textStream: ReadableStream<string>,\n    public readonly toolCallStream: ReadableStream<FunctionCall>,\n  ) {\n    this.id = shortuuid('item_');\n    this.generatedToolCalls = [];\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolOutput\nexport class _ToolOutput {\n  output: _JsOutput[];\n  firstToolFut: Future;\n\n  constructor() {\n    this.output = [];\n    this.firstToolFut = new Future();\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _SanitizedOutput {\n  toolCall: FunctionCall;\n  toolCallOutput?: FunctionCallOutput;\n  replyRequired: boolean;\n  agentTask?: Agent;\n\n  constructor(\n    toolCall: FunctionCall,\n    toolCallOutput: FunctionCallOutput | undefined,\n    replyRequired: boolean,\n    agentTask: Agent | undefined,\n  ) {\n    this.toolCall = toolCall;\n    this.toolCallOutput = toolCallOutput;\n    this.replyRequired = replyRequired;\n    this.agentTask = agentTask;\n  }\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    replyRequired?: boolean;\n    agentTask?: Agent;\n  }) {\n    const { toolCall, toolCallOutput, replyRequired = true, agentTask } = params;\n    return new _SanitizedOutput(toolCall, toolCallOutput, replyRequired, agentTask);\n  }\n}\n\nfunction isValidToolOutput(toolOutput: unknown): boolean {\n  const validTypes = ['string', 'number', 'boolean'];\n\n  if (validTypes.includes(typeof toolOutput)) {\n    return true;\n  }\n\n  if (toolOutput === undefined || toolOutput === null) {\n    return true;\n  }\n\n  if (Array.isArray(toolOutput)) {\n    return toolOutput.every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Set) {\n    return Array.from(toolOutput).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Map) {\n    return Array.from(toolOutput.values()).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Object) {\n    return Object.entries(toolOutput).every(\n      ([key, value]) => validTypes.includes(typeof key) && isValidToolOutput(value),\n    );\n  }\n\n  return false;\n}\n\nexport class ToolExecutionOutput {\n  constructor(\n    public readonly toolCall: FunctionCall,\n    public readonly toolCallOutput: FunctionCallOutput | undefined,\n    public readonly agentTask: Agent | undefined,\n    public readonly rawOutput: unknown,\n    public readonly rawException: Error | undefined,\n    public readonly replyRequired: boolean,\n  ) {}\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    agentTask?: Agent;\n    rawOutput: unknown;\n    rawException?: Error;\n    replyRequired?: boolean;\n  }) {\n    const {\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired = true,\n    } = params;\n    return new ToolExecutionOutput(\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired,\n    );\n  }\n}\n\nexport interface ToolOutput {\n  output: ToolExecutionOutput[];\n  firstToolStartedFuture: Future<void>;\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _JsOutput {\n  toolCall: FunctionCall;\n  output: unknown;\n  exception?: Error;\n\n  #logger = log();\n\n  constructor(toolCall: FunctionCall, output: unknown, exception: Error | undefined) {\n    this.toolCall = toolCall;\n    this.output = output;\n    this.exception = exception;\n  }\n\n  static create(params: { toolCall: FunctionCall; output?: unknown; exception?: Error }) {\n    const { toolCall, output = undefined, exception = undefined } = params;\n    return new _JsOutput(toolCall, output, exception);\n  }\n\n  sanitize(): _SanitizedOutput {\n    if (isToolError(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: this.exception.message,\n          isError: true,\n        }),\n      });\n    }\n\n    if (isStopResponse(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n      });\n    }\n\n    if (this.exception !== undefined) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: 'An internal error occurred while executing the tool.', // Don't send the actual error message, as it may contain sensitive information\n          isError: true,\n        }),\n      });\n    }\n\n    let agentTask: Agent | undefined = undefined;\n    let toolOutput: unknown = this.output;\n    if (isAgentHandoff(this.output)) {\n      agentTask = this.output.agent;\n      toolOutput = this.output.returns;\n    }\n\n    if (!isValidToolOutput(toolOutput)) {\n      this.#logger.error(\n        {\n          callId: this.toolCall.callId,\n          function: this.toolCall.name,\n        },\n        `AI function ${this.toolCall.name} returned an invalid output`,\n      );\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: undefined,\n      });\n    }\n\n    return _SanitizedOutput.create({\n      toolCall: FunctionCall.create({ ...this.toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: this.toolCall.name,\n        callId: this.toolCall.callId,\n        output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n        isError: false,\n      }),\n      replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n      agentTask,\n    });\n  }\n}\n\nexport function createToolOutput(params: {\n  toolCall: FunctionCall;\n  output?: unknown;\n  exception?: Error;\n}): ToolExecutionOutput {\n  const { toolCall, output, exception } = params;\n  const logger = log();\n\n  // support returning Exception instead of raising them (for devex purposes inside evals)\n  let finalOutput = output;\n  let finalException = exception;\n  if (output instanceof Error) {\n    finalException = output;\n    finalOutput = undefined;\n  }\n\n  if (isToolError(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: finalException.message,\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (isStopResponse(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (finalException !== undefined) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: 'An internal error occurred', // Don't send the actual error message, as it may contain sensitive information\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  let agentTask: Agent | undefined = undefined;\n  let toolOutput: unknown = finalOutput;\n  if (isAgentHandoff(finalOutput)) {\n    agentTask = finalOutput.agent;\n    toolOutput = finalOutput.returns;\n  }\n\n  if (!isValidToolOutput(toolOutput)) {\n    logger.error(\n      {\n        callId: toolCall.callId,\n        output: finalOutput,\n      },\n      `AI function ${toolCall.name} returned an invalid output`,\n    );\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  return ToolExecutionOutput.create({\n    toolCall: FunctionCall.create({ ...toolCall }),\n    toolCallOutput: FunctionCallOutput.create({\n      name: toolCall.name,\n      callId: toolCall.callId,\n      output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n      isError: false,\n    }),\n    replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n    agentTask,\n    rawOutput: finalOutput,\n    rawException: finalException,\n  });\n}\n\nconst INSTRUCTIONS_MESSAGE_ID = 'lk.agent_task.instructions';\n\n/**\n * Update the instruction message in the chat context or insert a new one if missing.\n *\n * This function looks for an existing instruction message in the chat context using the identifier\n * 'INSTRUCTIONS_MESSAGE_ID'.\n *\n * @param options - The options for updating the instructions.\n * @param options.chatCtx - The chat context to update.\n * @param options.instructions - The instructions to add.\n * @param options.addIfMissing - Whether to add the instructions if they are missing.\n */\nexport function updateInstructions(options: {\n  chatCtx: ChatContext;\n  instructions: string;\n  addIfMissing: boolean;\n}) {\n  const { chatCtx, instructions, addIfMissing } = options;\n\n  const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n  if (idx !== undefined) {\n    if (chatCtx.items[idx]!.type === 'message') {\n      // create a new instance to avoid mutating the original\n      chatCtx.items[idx] = ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n        createdAt: chatCtx.items[idx]!.createdAt,\n      });\n    } else {\n      throw new Error('expected the instructions inside the chatCtx to be of type \"message\"');\n    }\n  } else if (addIfMissing) {\n    // insert the instructions at the beginning of the chat context\n    chatCtx.items.unshift(\n      ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n      }),\n    );\n  }\n}\n\nexport function performLLMInference(\n  node: LLMNode,\n  chatCtx: ChatContext,\n  toolCtx: ToolContext,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, _LLMGenerationData] {\n  const textStream = new IdentityTransform<string>();\n  const toolCallStream = new IdentityTransform<FunctionCall>();\n\n  const textWriter = textStream.writable.getWriter();\n  const toolCallWriter = toolCallStream.writable.getWriter();\n  const data = new _LLMGenerationData(textStream.readable, toolCallStream.readable);\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let llmStreamReader: ReadableStreamDefaultReader<string | ChatChunk> | null = null;\n    let llmStream: ReadableStream<string | ChatChunk> | null = null;\n\n    try {\n      llmStream = await node(chatCtx, toolCtx, modelSettings);\n      if (llmStream === null) {\n        await textWriter.close();\n        return;\n      }\n\n      // TODO(brian): add support for dynamic tools\n\n      llmStreamReader = llmStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await llmStreamReader.read();\n        if (done) {\n          break;\n        }\n\n        if (typeof chunk === 'string') {\n          data.generatedText += chunk;\n          await textWriter.write(chunk);\n          // TODO(shubhra): better way to check??\n        } else {\n          if (chunk.delta === undefined) {\n            continue;\n          }\n\n          if (chunk.delta.toolCalls) {\n            for (const tool of chunk.delta.toolCalls) {\n              if (tool.type !== 'function_call') continue;\n\n              const toolCall = FunctionCall.create({\n                callId: `${data.id}/fnc_${data.generatedToolCalls.length}`,\n                name: tool.name,\n                args: tool.args,\n              });\n\n              data.generatedToolCalls.push(toolCall);\n              await toolCallWriter.write(toolCall);\n            }\n          }\n\n          if (chunk.delta.content) {\n            data.generatedText += chunk.delta.content;\n            await textWriter.write(chunk.delta.content);\n          }\n        }\n\n        // No need to check if chunk is of type other than ChatChunk or string like in\n        // Python since chunk is defined in the type ChatChunk | string in TypeScript\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      llmStreamReader?.releaseLock();\n      await llmStream?.cancel();\n      await textWriter.close();\n      await toolCallWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performLLMInference'),\n    data,\n  ];\n}\n\nexport function performTTSInference(\n  node: TTSNode,\n  text: ReadableStream<string>,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, ReadableStream<AudioFrame>] {\n  const audioStream = new IdentityTransform<AudioFrame>();\n  const outputWriter = audioStream.writable.getWriter();\n  const audioOutputStream = audioStream.readable;\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let ttsStreamReader: ReadableStreamDefaultReader<AudioFrame> | null = null;\n    let ttsStream: ReadableStream<AudioFrame> | null = null;\n\n    try {\n      ttsStream = await node(text, modelSettings);\n      if (ttsStream === null) {\n        await outputWriter.close();\n        return;\n      }\n\n      ttsStreamReader = ttsStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await ttsStreamReader.read();\n        if (done) {\n          break;\n        }\n        await outputWriter.write(chunk);\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      ttsStreamReader?.releaseLock();\n      await ttsStream?.cancel();\n      await outputWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performTTSInference'),\n    audioOutputStream,\n  ];\n}\n\nexport interface _TextOut {\n  text: string;\n  firstTextFut: Future;\n}\n\nasync function forwardText(\n  source: ReadableStream<string>,\n  out: _TextOut,\n  signal: AbortSignal,\n  textOutput: TextOutput | null,\n): Promise<void> {\n  const reader = source.getReader();\n  try {\n    while (true) {\n      if (signal.aborted) {\n        break;\n      }\n      const { done, value: delta } = await reader.read();\n      if (done) break;\n      out.text += delta;\n      if (textOutput !== null) {\n        await textOutput.captureText(delta);\n      }\n      if (!out.firstTextFut.done) {\n        out.firstTextFut.resolve();\n      }\n    }\n  } finally {\n    if (textOutput !== null) {\n      textOutput.flush();\n    }\n    reader?.releaseLock();\n  }\n}\n\nexport function performTextForwarding(\n  source: ReadableStream<string>,\n  controller: AbortController,\n  textOutput: TextOutput | null,\n): [Task<void>, _TextOut] {\n  const out = {\n    text: '',\n    firstTextFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardText(source, out, controller.signal, textOutput),\n      controller,\n      'performTextForwarding',\n    ),\n    out,\n  ];\n}\n\nexport interface _AudioOut {\n  audio: Array<AudioFrame>;\n  firstFrameFut: Future;\n}\n\nasync function forwardAudio(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOuput: AudioOutput,\n  out: _AudioOut,\n  signal?: AbortSignal,\n): Promise<void> {\n  const reader = ttsStream.getReader();\n  let resampler: AudioResampler | null = null;\n\n  try {\n    while (true) {\n      if (signal?.aborted) {\n        break;\n      }\n\n      const { done, value: frame } = await reader.read();\n      if (done) break;\n\n      out.audio.push(frame);\n\n      if (\n        !out.firstFrameFut.done &&\n        audioOuput.sampleRate &&\n        audioOuput.sampleRate !== frame.sampleRate &&\n        !resampler\n      ) {\n        resampler = new AudioResampler(frame.sampleRate, audioOuput.sampleRate, 1);\n      }\n\n      if (resampler) {\n        for (const f of resampler.push(frame)) {\n          await audioOuput.captureFrame(f);\n        }\n      } else {\n        await audioOuput.captureFrame(frame);\n      }\n\n      // set the first frame future if not already set\n      // (after completing the first frame)\n      if (!out.firstFrameFut.done) {\n        out.firstFrameFut.resolve();\n      }\n    }\n  } finally {\n    reader?.releaseLock();\n    if (resampler) {\n      for (const f of resampler.flush()) {\n        await audioOuput.captureFrame(f);\n      }\n    }\n    audioOuput.flush();\n  }\n}\n\nexport function performAudioForwarding(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOutput: AudioOutput,\n  controller: AbortController,\n): [Task<void>, _AudioOut] {\n  const out = {\n    audio: [],\n    firstFrameFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardAudio(ttsStream, audioOutput, out, controller.signal),\n      controller,\n      'performAudioForwarding',\n    ),\n    out,\n  ];\n}\n\nexport function performToolExecutions({\n  session,\n  speechHandle,\n  toolCtx,\n  toolChoice,\n  toolCallStream,\n  onToolExecutionStarted = () => {},\n  onToolExecutionCompleted = () => {},\n  controller,\n}: {\n  session: AgentSession;\n  speechHandle: SpeechHandle;\n  toolCtx: ToolContext;\n  toolChoice?: ToolChoice;\n  toolCallStream: ReadableStream<FunctionCall>;\n  onToolExecutionStarted?: (toolCall: FunctionCall) => void;\n  onToolExecutionCompleted?: (toolExecutionOutput: ToolExecutionOutput) => void;\n  controller: AbortController;\n}): [Task<void>, ToolOutput] {\n  const logger = log();\n  const toolOutput: ToolOutput = {\n    output: [],\n    firstToolStartedFuture: new Future(),\n  };\n\n  const toolCompleted = (out: ToolExecutionOutput) => {\n    onToolExecutionCompleted(out);\n    toolOutput.output.push(out);\n  };\n\n  const executeToolsTask = async (controller: AbortController) => {\n    const signal = controller.signal;\n    const reader = toolCallStream.getReader();\n\n    const tasks: Promise<any>[] = [];\n    while (!signal.aborted) {\n      const { done, value: toolCall } = await reader.read();\n      if (signal.aborted) break;\n      if (done) break;\n\n      if (toolChoice === 'none') {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          \"received a tool call with toolChoice set to 'none', ignoring\",\n        );\n        continue;\n      }\n\n      // TODO(brian): assert other toolChoice values\n\n      const tool = toolCtx[toolCall.name];\n      if (!tool) {\n        logger.warn(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown AI function ${toolCall.name}`,\n        );\n        continue;\n      }\n\n      if (!isFunctionTool(tool)) {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown tool type: ${typeof tool}`,\n        );\n        continue;\n      }\n\n      let parsedArgs: object | undefined;\n\n      // Ensure valid arguments\n      try {\n        const jsonArgs = JSON.parse(toolCall.args);\n\n        if (tool.parameters instanceof ZodObject) {\n          parsedArgs = tool.parameters.parse(jsonArgs);\n        } else {\n          parsedArgs = jsonArgs;\n        }\n      } catch (rawError) {\n        const error = toError(rawError);\n        logger.error(\n          {\n            function: toolCall.name,\n            arguments: toolCall.args,\n            speech_id: speechHandle.id,\n            error: error.message,\n          },\n          `tried to call AI function ${toolCall.name} with invalid arguments`,\n        );\n        toolCompleted(\n          createToolOutput({\n            toolCall,\n            exception: error,\n          }),\n        );\n        continue;\n      }\n\n      if (!toolOutput.firstToolStartedFuture.done) {\n        toolOutput.firstToolStartedFuture.resolve();\n      }\n\n      onToolExecutionStarted(toolCall);\n\n      logger.info(\n        {\n          function: toolCall.name,\n          arguments: parsedArgs,\n          speech_id: speechHandle.id,\n        },\n        'Executing LLM tool call',\n      );\n\n      const toolExecution = asyncLocalStorage.run({ functionCall: toolCall }, async () => {\n        return await tool.execute(parsedArgs, {\n          ctx: new RunContext(session, speechHandle, toolCall),\n          toolCallId: toolCall.callId,\n          abortSignal: signal,\n        });\n      });\n\n      const tracableToolExecution = async (toolExecTask: Promise<unknown>) => {\n        // TODO(brian): add tracing\n\n        // await for task to complete, if task is aborted, set exception\n        let toolOutput: ToolExecutionOutput | undefined;\n        try {\n          const { result, isAborted } = await waitUntilAborted(toolExecTask, signal);\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: isAborted ? new Error('tool call was aborted') : undefined,\n            output: isAborted ? undefined : result,\n          });\n        } catch (rawError) {\n          logger.error(\n            {\n              function: toolCall.name,\n              speech_id: speechHandle.id,\n              error: toError(rawError).message,\n            },\n            'exception occurred while executing tool',\n          );\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: toError(rawError),\n          });\n        } finally {\n          if (!toolOutput) throw new Error('toolOutput is undefined');\n          toolCompleted(toolOutput);\n        }\n      };\n\n      // wait, not cancelling all tool calling tasks\n      tasks.push(tracableToolExecution(toolExecution));\n    }\n\n    await Promise.allSettled(tasks);\n    if (toolOutput.output.length > 0) {\n      logger.debug(\n        {\n          speech_id: speechHandle.id,\n        },\n        'tools execution completed',\n      );\n    }\n  };\n\n  return [Task.from(executeToolsTask, controller, 'performToolExecutions'), toolOutput];\n}\n\ntype Aborted<T> =\n  | {\n      result: T;\n      isAborted: false;\n    }\n  | {\n      result: undefined;\n      isAborted: true;\n    };\n\nasync function waitUntilAborted<T>(promise: Promise<T>, signal: AbortSignal): Promise<Aborted<T>> {\n  const abortFut = new Future<Aborted<T>>();\n\n  const resolveAbort = () => {\n    if (!abortFut.done) {\n      abortFut.resolve({ result: undefined, isAborted: true });\n    }\n  };\n\n  signal.addEventListener('abort', resolveAbort);\n\n  promise\n    .then((r) => {\n      if (!abortFut.done) {\n        abortFut.resolve({ result: r, isAborted: false });\n      }\n    })\n    .catch((e) => {\n      if (!abortFut.done) {\n        abortFut.reject(e);\n      }\n    })\n    .finally(() => {\n      signal.removeEventListener('abort', resolveAbort);\n    });\n\n  return await abortFut.await;\n}\n\nexport function removeInstructions(chatCtx: ChatContext) {\n  // loop in case there are items with the same id (shouldn't happen!)\n  while (true) {\n    const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n    if (idx !== undefined) {\n      chatCtx.items.splice(idx, 1);\n    } else {\n      break;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAA+B;AAE/B,iBAA0B;AAC1B,0BAKO;AAEP,0BAMO;AACP,iBAAoB;AACpB,gCAAkC;AAClC,mBAAiD;AACjD,mBAAkF;AAGlF,yBAA2B;AAIpB,MAAM,mBAAmB;AAAA,EAK9B,YACkB,YACA,gBAChB;AAFgB;AACA;AAEhB,SAAK,SAAK,wBAAU,OAAO;AAC3B,SAAK,qBAAqB,CAAC;AAAA,EAC7B;AAAA,EAVA,gBAAwB;AAAA,EACxB;AAAA,EACA;AASF;AAGO,MAAM,YAAY;AAAA,EACvB;AAAA,EACA;AAAA,EAEA,cAAc;AACZ,SAAK,SAAS,CAAC;AACf,SAAK,eAAe,IAAI,oBAAO;AAAA,EACjC;AACF;AAGO,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,UACA,gBACA,eACA,WACA;AACA,SAAK,WAAW;AAChB,SAAK,iBAAiB;AACtB,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAKX;AACD,UAAM,EAAE,UAAU,gBAAgB,gBAAgB,MAAM,UAAU,IAAI;AACtE,WAAO,IAAI,iBAAiB,UAAU,gBAAgB,eAAe,SAAS;AAAA,EAChF;AACF;AAEA,SAAS,kBAAkB,YAA8B;AACvD,QAAM,aAAa,CAAC,UAAU,UAAU,SAAS;AAEjD,MAAI,WAAW,SAAS,OAAO,UAAU,GAAG;AAC1C,WAAO;AAAA,EACT;AAEA,MAAI,eAAe,UAAa,eAAe,MAAM;AACnD,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,WAAO,WAAW,MAAM,iBAAiB;AAAA,EAC3C;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,UAAU,EAAE,MAAM,iBAAiB;AAAA,EACvD;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,WAAW,OAAO,CAAC,EAAE,MAAM,iBAAiB;AAAA,EAChE;AAEA,MAAI,sBAAsB,QAAQ;AAChC,WAAO,OAAO,QAAQ,UAAU,EAAE;AAAA,MAChC,CAAC,CAAC,KAAK,KAAK,MAAM,WAAW,SAAS,OAAO,GAAG,KAAK,kBAAkB,KAAK;AAAA,IAC9E;AAAA,EACF;AAEA,SAAO;AACT;AAEO,MAAM,oBAAoB;AAAA,EAC/B,YACkB,UACA,gBACA,WACA,WACA,cACA,eAChB;AANgB;AACA;AACA;AACA;AACA;AACA;AAAA,EACf;AAAA,EAEH,OAAO,OAAO,QAOX;AACD,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,IAClB,IAAI;AACJ,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAQO,MAAM,UAAU;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,cAAU,gBAAI;AAAA,EAEd,YAAY,UAAwB,QAAiB,WAA8B;AACjF,SAAK,WAAW;AAChB,SAAK,SAAS;AACd,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAAyE;AACrF,UAAM,EAAE,UAAU,SAAS,QAAW,YAAY,OAAU,IAAI;AAChE,WAAO,IAAI,UAAU,UAAU,QAAQ,SAAS;AAAA,EAClD;AAAA,EAEA,WAA6B;AAC3B,YAAI,iCAAY,KAAK,SAAS,GAAG;AAC/B,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ,KAAK,UAAU;AAAA,UACvB,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,YAAI,6BAAe,KAAK,SAAS,GAAG;AAClC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MACpD,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,cAAc,QAAW;AAChC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ;AAAA;AAAA,UACR,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,QAAI,YAA+B;AACnC,QAAI,aAAsB,KAAK;AAC/B,YAAI,oCAAe,KAAK,MAAM,GAAG;AAC/B,kBAAY,KAAK,OAAO;AACxB,mBAAa,KAAK,OAAO;AAAA,IAC3B;AAEA,QAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK,SAAS;AAAA,UACtB,UAAU,KAAK,SAAS;AAAA,QAC1B;AAAA,QACA,eAAe,KAAK,SAAS,IAAI;AAAA,MACnC;AACA,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB;AAAA,MAClB,CAAC;AAAA,IACH;AAEA,WAAO,iBAAiB,OAAO;AAAA,MAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MAClD,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,KAAK,SAAS;AAAA,QACpB,QAAQ,KAAK,SAAS;AAAA,QACtB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,QAChE,SAAS;AAAA,MACX,CAAC;AAAA,MACD,eAAe,eAAe;AAAA;AAAA,MAC9B;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,SAAS,iBAAiB,QAIT;AACtB,QAAM,EAAE,UAAU,QAAQ,UAAU,IAAI;AACxC,QAAM,aAAS,gBAAI;AAGnB,MAAI,cAAc;AAClB,MAAI,iBAAiB;AACrB,MAAI,kBAAkB,OAAO;AAC3B,qBAAiB;AACjB,kBAAc;AAAA,EAChB;AAEA,UAAI,iCAAY,cAAc,GAAG;AAC/B,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ,eAAe;AAAA,QACvB,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,UAAI,6BAAe,cAAc,GAAG;AAClC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,mBAAmB,QAAW;AAChC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA;AAAA,QACR,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,YAA+B;AACnC,MAAI,aAAsB;AAC1B,UAAI,oCAAe,WAAW,GAAG;AAC/B,gBAAY,YAAY;AACxB,iBAAa,YAAY;AAAA,EAC3B;AAEA,MAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAO;AAAA,MACL;AAAA,QACE,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA,MACV;AAAA,MACA,eAAe,SAAS,IAAI;AAAA,IAC9B;AACA,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,SAAO,oBAAoB,OAAO;AAAA,IAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,IAC7C,gBAAgB,uCAAmB,OAAO;AAAA,MACxC,MAAM,SAAS;AAAA,MACf,QAAQ,SAAS;AAAA,MACjB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,MAChE,SAAS;AAAA,IACX,CAAC;AAAA,IACD,eAAe,eAAe;AAAA;AAAA,IAC9B;AAAA,IACA,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACH;AAEA,MAAM,0BAA0B;AAazB,SAAS,mBAAmB,SAIhC;AACD,QAAM,EAAE,SAAS,cAAc,aAAa,IAAI;AAEhD,QAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,MAAI,QAAQ,QAAW;AACrB,QAAI,QAAQ,MAAM,GAAG,EAAG,SAAS,WAAW;AAE1C,cAAQ,MAAM,GAAG,IAAI,gCAAY,OAAO;AAAA,QACtC,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,WAAW,QAAQ,MAAM,GAAG,EAAG;AAAA,MACjC,CAAC;AAAA,IACH,OAAO;AACL,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAAA,EACF,WAAW,cAAc;AAEvB,YAAQ,MAAM;AAAA,MACZ,gCAAY,OAAO;AAAA,QACjB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,MACxB,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,SACA,SACA,eACA,YACkC;AAClC,QAAM,aAAa,IAAI,4CAA0B;AACjD,QAAM,iBAAiB,IAAI,4CAAgC;AAE3D,QAAM,aAAa,WAAW,SAAS,UAAU;AACjD,QAAM,iBAAiB,eAAe,SAAS,UAAU;AACzD,QAAM,OAAO,IAAI,mBAAmB,WAAW,UAAU,eAAe,QAAQ;AAEhF,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAA0E;AAC9E,QAAI,YAAuD;AAE3D,QAAI;AACF,kBAAY,MAAM,KAAK,SAAS,SAAS,aAAa;AACtD,UAAI,cAAc,MAAM;AACtB,cAAM,WAAW,MAAM;AACvB;AAAA,MACF;AAIA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AAEA,YAAI,OAAO,UAAU,UAAU;AAC7B,eAAK,iBAAiB;AACtB,gBAAM,WAAW,MAAM,KAAK;AAAA,QAE9B,OAAO;AACL,cAAI,MAAM,UAAU,QAAW;AAC7B;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,WAAW;AACzB,uBAAW,QAAQ,MAAM,MAAM,WAAW;AACxC,kBAAI,KAAK,SAAS,gBAAiB;AAEnC,oBAAM,WAAW,iCAAa,OAAO;AAAA,gBACnC,QAAQ,GAAG,KAAK,EAAE,QAAQ,KAAK,mBAAmB,MAAM;AAAA,gBACxD,MAAM,KAAK;AAAA,gBACX,MAAM,KAAK;AAAA,cACb,CAAC;AAED,mBAAK,mBAAmB,KAAK,QAAQ;AACrC,oBAAM,eAAe,MAAM,QAAQ;AAAA,YACrC;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,SAAS;AACvB,iBAAK,iBAAiB,MAAM,MAAM;AAClC,kBAAM,WAAW,MAAM,MAAM,MAAM,OAAO;AAAA,UAC5C;AAAA,QACF;AAAA,MAIF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,WAAW,MAAM;AACvB,YAAM,eAAe,MAAM;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,MACA,eACA,YAC0C;AAC1C,QAAM,cAAc,IAAI,4CAA8B;AACtD,QAAM,eAAe,YAAY,SAAS,UAAU;AACpD,QAAM,oBAAoB,YAAY;AAEtC,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAAkE;AACtE,QAAI,YAA+C;AAEnD,QAAI;AACF,kBAAY,MAAM,KAAK,MAAM,aAAa;AAC1C,UAAI,cAAc,MAAM;AACtB,cAAM,aAAa,MAAM;AACzB;AAAA,MACF;AAEA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AACA,cAAM,aAAa,MAAM,KAAK;AAAA,MAChC;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,aAAa,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAOA,eAAe,YACb,QACA,KACA,QACA,YACe;AACf,QAAM,SAAS,OAAO,UAAU;AAChC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,OAAO,SAAS;AAClB;AAAA,MACF;AACA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AACV,UAAI,QAAQ;AACZ,UAAI,eAAe,MAAM;AACvB,cAAM,WAAW,YAAY,KAAK;AAAA,MACpC;AACA,UAAI,CAAC,IAAI,aAAa,MAAM;AAC1B,YAAI,aAAa,QAAQ;AAAA,MAC3B;AAAA,IACF;AAAA,EACF,UAAE;AACA,QAAI,eAAe,MAAM;AACvB,iBAAW,MAAM;AAAA,IACnB;AACA,qCAAQ;AAAA,EACV;AACF;AAEO,SAAS,sBACd,QACA,YACA,YACwB;AACxB,QAAM,MAAM;AAAA,IACV,MAAM;AAAA,IACN,cAAc,IAAI,oBAAO;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,YAAY,QAAQ,KAAKA,YAAW,QAAQ,UAAU;AAAA,MACtE;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAOA,eAAe,aACb,WACA,YACA,KACA,QACe;AACf,QAAM,SAAS,UAAU,UAAU;AACnC,MAAI,YAAmC;AAEvC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,iCAAQ,SAAS;AACnB;AAAA,MACF;AAEA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AAEV,UAAI,MAAM,KAAK,KAAK;AAEpB,UACE,CAAC,IAAI,cAAc,QACnB,WAAW,cACX,WAAW,eAAe,MAAM,cAChC,CAAC,WACD;AACA,oBAAY,IAAI,+BAAe,MAAM,YAAY,WAAW,YAAY,CAAC;AAAA,MAC3E;AAEA,UAAI,WAAW;AACb,mBAAW,KAAK,UAAU,KAAK,KAAK,GAAG;AACrC,gBAAM,WAAW,aAAa,CAAC;AAAA,QACjC;AAAA,MACF,OAAO;AACL,cAAM,WAAW,aAAa,KAAK;AAAA,MACrC;AAIA,UAAI,CAAC,IAAI,cAAc,MAAM;AAC3B,YAAI,cAAc,QAAQ;AAAA,MAC5B;AAAA,IACF;AAAA,EACF,UAAE;AACA,qCAAQ;AACR,QAAI,WAAW;AACb,iBAAW,KAAK,UAAU,MAAM,GAAG;AACjC,cAAM,WAAW,aAAa,CAAC;AAAA,MACjC;AAAA,IACF;AACA,eAAW,MAAM;AAAA,EACnB;AACF;AAEO,SAAS,uBACd,WACA,aACA,YACyB;AACzB,QAAM,MAAM;AAAA,IACV,OAAO,CAAC;AAAA,IACR,eAAe,IAAI,oBAAO;AAAA,EAC5B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,aAAa,WAAW,aAAa,KAAKA,YAAW,MAAM;AAAA,MAC3E;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAyB,MAAM;AAAA,EAAC;AAAA,EAChC,2BAA2B,MAAM;AAAA,EAAC;AAAA,EAClC;AACF,GAS6B;AAC3B,QAAM,aAAS,gBAAI;AACnB,QAAM,aAAyB;AAAA,IAC7B,QAAQ,CAAC;AAAA,IACT,wBAAwB,IAAI,oBAAO;AAAA,EACrC;AAEA,QAAM,gBAAgB,CAAC,QAA6B;AAClD,6BAAyB,GAAG;AAC5B,eAAW,OAAO,KAAK,GAAG;AAAA,EAC5B;AAEA,QAAM,mBAAmB,OAAOA,gBAAgC;AAC9D,UAAM,SAASA,YAAW;AAC1B,UAAM,SAAS,eAAe,UAAU;AAExC,UAAM,QAAwB,CAAC;AAC/B,WAAO,CAAC,OAAO,SAAS;AACtB,YAAM,EAAE,MAAM,OAAO,SAAS,IAAI,MAAM,OAAO,KAAK;AACpD,UAAI,OAAO,QAAS;AACpB,UAAI,KAAM;AAEV,UAAI,eAAe,QAAQ;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA;AAAA,QACF;AACA;AAAA,MACF;AAIA,YAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,UAAI,CAAC,MAAM;AACT,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,uBAAuB,SAAS,IAAI;AAAA,QACtC;AACA;AAAA,MACF;AAEA,UAAI,KAAC,oCAAe,IAAI,GAAG;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,sBAAsB,OAAO,IAAI;AAAA,QACnC;AACA;AAAA,MACF;AAEA,UAAI;AAGJ,UAAI;AACF,cAAM,WAAW,KAAK,MAAM,SAAS,IAAI;AAEzC,YAAI,KAAK,sBAAsB,sBAAW;AACxC,uBAAa,KAAK,WAAW,MAAM,QAAQ;AAAA,QAC7C,OAAO;AACL,uBAAa;AAAA,QACf;AAAA,MACF,SAAS,UAAU;AACjB,cAAM,YAAQ,sBAAQ,QAAQ;AAC9B,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,SAAS;AAAA,YACpB,WAAW,aAAa;AAAA,YACxB,OAAO,MAAM;AAAA,UACf;AAAA,UACA,6BAA6B,SAAS,IAAI;AAAA,QAC5C;AACA;AAAA,UACE,iBAAiB;AAAA,YACf;AAAA,YACA,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,UAAI,CAAC,WAAW,uBAAuB,MAAM;AAC3C,mBAAW,uBAAuB,QAAQ;AAAA,MAC5C;AAEA,6BAAuB,QAAQ;AAE/B,aAAO;AAAA,QACL;AAAA,UACE,UAAU,SAAS;AAAA,UACnB,WAAW;AAAA,UACX,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAEA,YAAM,gBAAgB,+BAAkB,IAAI,EAAE,cAAc,SAAS,GAAG,YAAY;AAClF,eAAO,MAAM,KAAK,QAAQ,YAAY;AAAA,UACpC,KAAK,IAAI,8BAAW,SAAS,cAAc,QAAQ;AAAA,UACnD,YAAY,SAAS;AAAA,UACrB,aAAa;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAED,YAAM,wBAAwB,OAAO,iBAAmC;AAItE,YAAIC;AACJ,YAAI;AACF,gBAAM,EAAE,QAAQ,UAAU,IAAI,MAAM,iBAAiB,cAAc,MAAM;AACzE,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,WAAW,YAAY,IAAI,MAAM,uBAAuB,IAAI;AAAA,YAC5D,QAAQ,YAAY,SAAY;AAAA,UAClC,CAAC;AAAA,QACH,SAAS,UAAU;AACjB,iBAAO;AAAA,YACL;AAAA,cACE,UAAU,SAAS;AAAA,cACnB,WAAW,aAAa;AAAA,cACxB,WAAO,sBAAQ,QAAQ,EAAE;AAAA,YAC3B;AAAA,YACA;AAAA,UACF;AACA,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,eAAW,sBAAQ,QAAQ;AAAA,UAC7B,CAAC;AAAA,QACH,UAAE;AACA,cAAI,CAACA,YAAY,OAAM,IAAI,MAAM,yBAAyB;AAC1D,wBAAcA,WAAU;AAAA,QAC1B;AAAA,MACF;AAGA,YAAM,KAAK,sBAAsB,aAAa,CAAC;AAAA,IACjD;AAEA,UAAM,QAAQ,WAAW,KAAK;AAC9B,QAAI,WAAW,OAAO,SAAS,GAAG;AAChC,aAAO;AAAA,QACL;AAAA,UACE,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO,CAAC,kBAAK,KAAK,kBAAkB,YAAY,uBAAuB,GAAG,UAAU;AACtF;AAYA,eAAe,iBAAoB,SAAqB,QAA0C;AAChG,QAAM,WAAW,IAAI,oBAAmB;AAExC,QAAM,eAAe,MAAM;AACzB,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,QAAW,WAAW,KAAK,CAAC;AAAA,IACzD;AAAA,EACF;AAEA,SAAO,iBAAiB,SAAS,YAAY;AAE7C,UACG,KAAK,CAAC,MAAM;AACX,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,GAAG,WAAW,MAAM,CAAC;AAAA,IAClD;AAAA,EACF,CAAC,EACA,MAAM,CAAC,MAAM;AACZ,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,OAAO,CAAC;AAAA,IACnB;AAAA,EACF,CAAC,EACA,QAAQ,MAAM;AACb,WAAO,oBAAoB,SAAS,YAAY;AAAA,EAClD,CAAC;AAEH,SAAO,MAAM,SAAS;AACxB;AAEO,SAAS,mBAAmB,SAAsB;AAEvD,SAAO,MAAM;AACX,UAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,QAAI,QAAQ,QAAW;AACrB,cAAQ,MAAM,OAAO,KAAK,CAAC;AAAA,IAC7B,OAAO;AACL;AAAA,IACF;AAAA,EACF;AACF;","names":["controller","toolOutput"]}
+{"version":3,"sources":["../../src/voice/generation.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AudioResampler } from '@livekit/rtc-node';\nimport type { ReadableStream, ReadableStreamDefaultReader } from 'stream/web';\nimport {\n  type ChatContext,\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n} from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport {\n  type ToolChoice,\n  type ToolContext,\n  isAgentHandoff,\n  isFunctionTool,\n  isToolError,\n} from '../llm/tool_context.js';\nimport { isZodSchema, parseZodSchema } from '../llm/zod-utils.js';\nimport { log } from '../log.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { Future, Task, shortuuid, toError } from '../utils.js';\nimport { type Agent, type ModelSettings, asyncLocalStorage, isStopResponse } from './agent.js';\nimport type { AgentSession } from './agent_session.js';\nimport type { AudioOutput, LLMNode, TTSNode, TextOutput } from './io.js';\nimport { RunContext } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\n/** @internal */\nexport class _LLMGenerationData {\n  generatedText: string = '';\n  generatedToolCalls: FunctionCall[];\n  id: string;\n\n  constructor(\n    public readonly textStream: ReadableStream<string>,\n    public readonly toolCallStream: ReadableStream<FunctionCall>,\n  ) {\n    this.id = shortuuid('item_');\n    this.generatedToolCalls = [];\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolOutput\nexport class _ToolOutput {\n  output: _JsOutput[];\n  firstToolFut: Future;\n\n  constructor() {\n    this.output = [];\n    this.firstToolFut = new Future();\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _SanitizedOutput {\n  toolCall: FunctionCall;\n  toolCallOutput?: FunctionCallOutput;\n  replyRequired: boolean;\n  agentTask?: Agent;\n\n  constructor(\n    toolCall: FunctionCall,\n    toolCallOutput: FunctionCallOutput | undefined,\n    replyRequired: boolean,\n    agentTask: Agent | undefined,\n  ) {\n    this.toolCall = toolCall;\n    this.toolCallOutput = toolCallOutput;\n    this.replyRequired = replyRequired;\n    this.agentTask = agentTask;\n  }\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    replyRequired?: boolean;\n    agentTask?: Agent;\n  }) {\n    const { toolCall, toolCallOutput, replyRequired = true, agentTask } = params;\n    return new _SanitizedOutput(toolCall, toolCallOutput, replyRequired, agentTask);\n  }\n}\n\nfunction isValidToolOutput(toolOutput: unknown): boolean {\n  const validTypes = ['string', 'number', 'boolean'];\n\n  if (validTypes.includes(typeof toolOutput)) {\n    return true;\n  }\n\n  if (toolOutput === undefined || toolOutput === null) {\n    return true;\n  }\n\n  if (Array.isArray(toolOutput)) {\n    return toolOutput.every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Set) {\n    return Array.from(toolOutput).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Map) {\n    return Array.from(toolOutput.values()).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Object) {\n    return Object.entries(toolOutput).every(\n      ([key, value]) => validTypes.includes(typeof key) && isValidToolOutput(value),\n    );\n  }\n\n  return false;\n}\n\nexport class ToolExecutionOutput {\n  constructor(\n    public readonly toolCall: FunctionCall,\n    public readonly toolCallOutput: FunctionCallOutput | undefined,\n    public readonly agentTask: Agent | undefined,\n    public readonly rawOutput: unknown,\n    public readonly rawException: Error | undefined,\n    public readonly replyRequired: boolean,\n  ) {}\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    agentTask?: Agent;\n    rawOutput: unknown;\n    rawException?: Error;\n    replyRequired?: boolean;\n  }) {\n    const {\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired = true,\n    } = params;\n    return new ToolExecutionOutput(\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired,\n    );\n  }\n}\n\nexport interface ToolOutput {\n  output: ToolExecutionOutput[];\n  firstToolStartedFuture: Future<void>;\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _JsOutput {\n  toolCall: FunctionCall;\n  output: unknown;\n  exception?: Error;\n\n  #logger = log();\n\n  constructor(toolCall: FunctionCall, output: unknown, exception: Error | undefined) {\n    this.toolCall = toolCall;\n    this.output = output;\n    this.exception = exception;\n  }\n\n  static create(params: { toolCall: FunctionCall; output?: unknown; exception?: Error }) {\n    const { toolCall, output = undefined, exception = undefined } = params;\n    return new _JsOutput(toolCall, output, exception);\n  }\n\n  sanitize(): _SanitizedOutput {\n    if (isToolError(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: this.exception.message,\n          isError: true,\n        }),\n      });\n    }\n\n    if (isStopResponse(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n      });\n    }\n\n    if (this.exception !== undefined) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: 'An internal error occurred while executing the tool.', // Don't send the actual error message, as it may contain sensitive information\n          isError: true,\n        }),\n      });\n    }\n\n    let agentTask: Agent | undefined = undefined;\n    let toolOutput: unknown = this.output;\n    if (isAgentHandoff(this.output)) {\n      agentTask = this.output.agent;\n      toolOutput = this.output.returns;\n    }\n\n    if (!isValidToolOutput(toolOutput)) {\n      this.#logger.error(\n        {\n          callId: this.toolCall.callId,\n          function: this.toolCall.name,\n        },\n        `AI function ${this.toolCall.name} returned an invalid output`,\n      );\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: undefined,\n      });\n    }\n\n    return _SanitizedOutput.create({\n      toolCall: FunctionCall.create({ ...this.toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: this.toolCall.name,\n        callId: this.toolCall.callId,\n        output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n        isError: false,\n      }),\n      replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n      agentTask,\n    });\n  }\n}\n\nexport function createToolOutput(params: {\n  toolCall: FunctionCall;\n  output?: unknown;\n  exception?: Error;\n}): ToolExecutionOutput {\n  const { toolCall, output, exception } = params;\n  const logger = log();\n\n  // support returning Exception instead of raising them (for devex purposes inside evals)\n  let finalOutput = output;\n  let finalException = exception;\n  if (output instanceof Error) {\n    finalException = output;\n    finalOutput = undefined;\n  }\n\n  if (isToolError(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: finalException.message,\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (isStopResponse(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (finalException !== undefined) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: 'An internal error occurred', // Don't send the actual error message, as it may contain sensitive information\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  let agentTask: Agent | undefined = undefined;\n  let toolOutput: unknown = finalOutput;\n  if (isAgentHandoff(finalOutput)) {\n    agentTask = finalOutput.agent;\n    toolOutput = finalOutput.returns;\n  }\n\n  if (!isValidToolOutput(toolOutput)) {\n    logger.error(\n      {\n        callId: toolCall.callId,\n        output: finalOutput,\n      },\n      `AI function ${toolCall.name} returned an invalid output`,\n    );\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  return ToolExecutionOutput.create({\n    toolCall: FunctionCall.create({ ...toolCall }),\n    toolCallOutput: FunctionCallOutput.create({\n      name: toolCall.name,\n      callId: toolCall.callId,\n      output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n      isError: false,\n    }),\n    replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n    agentTask,\n    rawOutput: finalOutput,\n    rawException: finalException,\n  });\n}\n\nconst INSTRUCTIONS_MESSAGE_ID = 'lk.agent_task.instructions';\n\n/**\n * Update the instruction message in the chat context or insert a new one if missing.\n *\n * This function looks for an existing instruction message in the chat context using the identifier\n * 'INSTRUCTIONS_MESSAGE_ID'.\n *\n * @param options - The options for updating the instructions.\n * @param options.chatCtx - The chat context to update.\n * @param options.instructions - The instructions to add.\n * @param options.addIfMissing - Whether to add the instructions if they are missing.\n */\nexport function updateInstructions(options: {\n  chatCtx: ChatContext;\n  instructions: string;\n  addIfMissing: boolean;\n}) {\n  const { chatCtx, instructions, addIfMissing } = options;\n\n  const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n  if (idx !== undefined) {\n    if (chatCtx.items[idx]!.type === 'message') {\n      // create a new instance to avoid mutating the original\n      chatCtx.items[idx] = ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n        createdAt: chatCtx.items[idx]!.createdAt,\n      });\n    } else {\n      throw new Error('expected the instructions inside the chatCtx to be of type \"message\"');\n    }\n  } else if (addIfMissing) {\n    // insert the instructions at the beginning of the chat context\n    chatCtx.items.unshift(\n      ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n      }),\n    );\n  }\n}\n\nexport function performLLMInference(\n  node: LLMNode,\n  chatCtx: ChatContext,\n  toolCtx: ToolContext,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, _LLMGenerationData] {\n  const textStream = new IdentityTransform<string>();\n  const toolCallStream = new IdentityTransform<FunctionCall>();\n\n  const textWriter = textStream.writable.getWriter();\n  const toolCallWriter = toolCallStream.writable.getWriter();\n  const data = new _LLMGenerationData(textStream.readable, toolCallStream.readable);\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let llmStreamReader: ReadableStreamDefaultReader<string | ChatChunk> | null = null;\n    let llmStream: ReadableStream<string | ChatChunk> | null = null;\n\n    try {\n      llmStream = await node(chatCtx, toolCtx, modelSettings);\n      if (llmStream === null) {\n        await textWriter.close();\n        return;\n      }\n\n      // TODO(brian): add support for dynamic tools\n\n      llmStreamReader = llmStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await llmStreamReader.read();\n        if (done) {\n          break;\n        }\n\n        if (typeof chunk === 'string') {\n          data.generatedText += chunk;\n          await textWriter.write(chunk);\n          // TODO(shubhra): better way to check??\n        } else {\n          if (chunk.delta === undefined) {\n            continue;\n          }\n\n          if (chunk.delta.toolCalls) {\n            for (const tool of chunk.delta.toolCalls) {\n              if (tool.type !== 'function_call') continue;\n\n              const toolCall = FunctionCall.create({\n                callId: `${data.id}/fnc_${data.generatedToolCalls.length}`,\n                name: tool.name,\n                args: tool.args,\n              });\n\n              data.generatedToolCalls.push(toolCall);\n              await toolCallWriter.write(toolCall);\n            }\n          }\n\n          if (chunk.delta.content) {\n            data.generatedText += chunk.delta.content;\n            await textWriter.write(chunk.delta.content);\n          }\n        }\n\n        // No need to check if chunk is of type other than ChatChunk or string like in\n        // Python since chunk is defined in the type ChatChunk | string in TypeScript\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      llmStreamReader?.releaseLock();\n      await llmStream?.cancel();\n      await textWriter.close();\n      await toolCallWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performLLMInference'),\n    data,\n  ];\n}\n\nexport function performTTSInference(\n  node: TTSNode,\n  text: ReadableStream<string>,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, ReadableStream<AudioFrame>] {\n  const audioStream = new IdentityTransform<AudioFrame>();\n  const outputWriter = audioStream.writable.getWriter();\n  const audioOutputStream = audioStream.readable;\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let ttsStreamReader: ReadableStreamDefaultReader<AudioFrame> | null = null;\n    let ttsStream: ReadableStream<AudioFrame> | null = null;\n\n    try {\n      ttsStream = await node(text, modelSettings);\n      if (ttsStream === null) {\n        await outputWriter.close();\n        return;\n      }\n\n      ttsStreamReader = ttsStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await ttsStreamReader.read();\n        if (done) {\n          break;\n        }\n        await outputWriter.write(chunk);\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      ttsStreamReader?.releaseLock();\n      await ttsStream?.cancel();\n      await outputWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performTTSInference'),\n    audioOutputStream,\n  ];\n}\n\nexport interface _TextOut {\n  text: string;\n  firstTextFut: Future;\n}\n\nasync function forwardText(\n  source: ReadableStream<string>,\n  out: _TextOut,\n  signal: AbortSignal,\n  textOutput: TextOutput | null,\n): Promise<void> {\n  const reader = source.getReader();\n  try {\n    while (true) {\n      if (signal.aborted) {\n        break;\n      }\n      const { done, value: delta } = await reader.read();\n      if (done) break;\n      out.text += delta;\n      if (textOutput !== null) {\n        await textOutput.captureText(delta);\n      }\n      if (!out.firstTextFut.done) {\n        out.firstTextFut.resolve();\n      }\n    }\n  } finally {\n    if (textOutput !== null) {\n      textOutput.flush();\n    }\n    reader?.releaseLock();\n  }\n}\n\nexport function performTextForwarding(\n  source: ReadableStream<string>,\n  controller: AbortController,\n  textOutput: TextOutput | null,\n): [Task<void>, _TextOut] {\n  const out = {\n    text: '',\n    firstTextFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardText(source, out, controller.signal, textOutput),\n      controller,\n      'performTextForwarding',\n    ),\n    out,\n  ];\n}\n\nexport interface _AudioOut {\n  audio: Array<AudioFrame>;\n  firstFrameFut: Future;\n}\n\nasync function forwardAudio(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOuput: AudioOutput,\n  out: _AudioOut,\n  signal?: AbortSignal,\n): Promise<void> {\n  const reader = ttsStream.getReader();\n  let resampler: AudioResampler | null = null;\n\n  try {\n    while (true) {\n      if (signal?.aborted) {\n        break;\n      }\n\n      const { done, value: frame } = await reader.read();\n      if (done) break;\n\n      out.audio.push(frame);\n\n      if (\n        !out.firstFrameFut.done &&\n        audioOuput.sampleRate &&\n        audioOuput.sampleRate !== frame.sampleRate &&\n        !resampler\n      ) {\n        resampler = new AudioResampler(frame.sampleRate, audioOuput.sampleRate, 1);\n      }\n\n      if (resampler) {\n        for (const f of resampler.push(frame)) {\n          await audioOuput.captureFrame(f);\n        }\n      } else {\n        await audioOuput.captureFrame(frame);\n      }\n\n      // set the first frame future if not already set\n      // (after completing the first frame)\n      if (!out.firstFrameFut.done) {\n        out.firstFrameFut.resolve();\n      }\n    }\n  } finally {\n    reader?.releaseLock();\n    if (resampler) {\n      for (const f of resampler.flush()) {\n        await audioOuput.captureFrame(f);\n      }\n    }\n    audioOuput.flush();\n  }\n}\n\nexport function performAudioForwarding(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOutput: AudioOutput,\n  controller: AbortController,\n): [Task<void>, _AudioOut] {\n  const out = {\n    audio: [],\n    firstFrameFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardAudio(ttsStream, audioOutput, out, controller.signal),\n      controller,\n      'performAudioForwarding',\n    ),\n    out,\n  ];\n}\n\nexport function performToolExecutions({\n  session,\n  speechHandle,\n  toolCtx,\n  toolChoice,\n  toolCallStream,\n  onToolExecutionStarted = () => {},\n  onToolExecutionCompleted = () => {},\n  controller,\n}: {\n  session: AgentSession;\n  speechHandle: SpeechHandle;\n  toolCtx: ToolContext;\n  toolChoice?: ToolChoice;\n  toolCallStream: ReadableStream<FunctionCall>;\n  onToolExecutionStarted?: (toolCall: FunctionCall) => void;\n  onToolExecutionCompleted?: (toolExecutionOutput: ToolExecutionOutput) => void;\n  controller: AbortController;\n}): [Task<void>, ToolOutput] {\n  const logger = log();\n  const toolOutput: ToolOutput = {\n    output: [],\n    firstToolStartedFuture: new Future(),\n  };\n\n  const toolCompleted = (out: ToolExecutionOutput) => {\n    onToolExecutionCompleted(out);\n    toolOutput.output.push(out);\n  };\n\n  const executeToolsTask = async (controller: AbortController) => {\n    const signal = controller.signal;\n    const reader = toolCallStream.getReader();\n\n    const tasks: Promise<any>[] = [];\n    while (!signal.aborted) {\n      const { done, value: toolCall } = await reader.read();\n      if (signal.aborted) break;\n      if (done) break;\n\n      if (toolChoice === 'none') {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          \"received a tool call with toolChoice set to 'none', ignoring\",\n        );\n        continue;\n      }\n\n      // TODO(brian): assert other toolChoice values\n\n      const tool = toolCtx[toolCall.name];\n      if (!tool) {\n        logger.warn(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown AI function ${toolCall.name}`,\n        );\n        continue;\n      }\n\n      if (!isFunctionTool(tool)) {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown tool type: ${typeof tool}`,\n        );\n        continue;\n      }\n\n      let parsedArgs: object | undefined;\n\n      // Ensure valid arguments\n      try {\n        const jsonArgs = JSON.parse(toolCall.args);\n\n        if (isZodSchema(tool.parameters)) {\n          const result = await parseZodSchema<object>(tool.parameters, jsonArgs);\n          if (result.success) {\n            parsedArgs = result.data;\n          } else {\n            throw result.error;\n          }\n        } else {\n          parsedArgs = jsonArgs;\n        }\n      } catch (rawError) {\n        const error = toError(rawError);\n        logger.error(\n          {\n            function: toolCall.name,\n            arguments: toolCall.args,\n            speech_id: speechHandle.id,\n            error: error.message,\n          },\n          `tried to call AI function ${toolCall.name} with invalid arguments`,\n        );\n        toolCompleted(\n          createToolOutput({\n            toolCall,\n            exception: error,\n          }),\n        );\n        continue;\n      }\n\n      if (!toolOutput.firstToolStartedFuture.done) {\n        toolOutput.firstToolStartedFuture.resolve();\n      }\n\n      onToolExecutionStarted(toolCall);\n\n      logger.info(\n        {\n          function: toolCall.name,\n          arguments: parsedArgs,\n          speech_id: speechHandle.id,\n        },\n        'Executing LLM tool call',\n      );\n\n      const toolExecution = asyncLocalStorage.run({ functionCall: toolCall }, async () => {\n        return await tool.execute(parsedArgs, {\n          ctx: new RunContext(session, speechHandle, toolCall),\n          toolCallId: toolCall.callId,\n          abortSignal: signal,\n        });\n      });\n\n      const tracableToolExecution = async (toolExecTask: Promise<unknown>) => {\n        // TODO(brian): add tracing\n\n        // await for task to complete, if task is aborted, set exception\n        let toolOutput: ToolExecutionOutput | undefined;\n        try {\n          const { result, isAborted } = await waitUntilAborted(toolExecTask, signal);\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: isAborted ? new Error('tool call was aborted') : undefined,\n            output: isAborted ? undefined : result,\n          });\n        } catch (rawError) {\n          logger.error(\n            {\n              function: toolCall.name,\n              speech_id: speechHandle.id,\n              error: toError(rawError).message,\n            },\n            'exception occurred while executing tool',\n          );\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: toError(rawError),\n          });\n        } finally {\n          if (!toolOutput) throw new Error('toolOutput is undefined');\n          toolCompleted(toolOutput);\n        }\n      };\n\n      // wait, not cancelling all tool calling tasks\n      tasks.push(tracableToolExecution(toolExecution));\n    }\n\n    await Promise.allSettled(tasks);\n    if (toolOutput.output.length > 0) {\n      logger.debug(\n        {\n          speech_id: speechHandle.id,\n        },\n        'tools execution completed',\n      );\n    }\n  };\n\n  return [Task.from(executeToolsTask, controller, 'performToolExecutions'), toolOutput];\n}\n\ntype Aborted<T> =\n  | {\n      result: T;\n      isAborted: false;\n    }\n  | {\n      result: undefined;\n      isAborted: true;\n    };\n\nasync function waitUntilAborted<T>(promise: Promise<T>, signal: AbortSignal): Promise<Aborted<T>> {\n  const abortFut = new Future<Aborted<T>>();\n\n  const resolveAbort = () => {\n    if (!abortFut.done) {\n      abortFut.resolve({ result: undefined, isAborted: true });\n    }\n  };\n\n  signal.addEventListener('abort', resolveAbort);\n\n  promise\n    .then((r) => {\n      if (!abortFut.done) {\n        abortFut.resolve({ result: r, isAborted: false });\n      }\n    })\n    .catch((e) => {\n      if (!abortFut.done) {\n        abortFut.reject(e);\n      }\n    })\n    .finally(() => {\n      signal.removeEventListener('abort', resolveAbort);\n    });\n\n  return await abortFut.await;\n}\n\nexport function removeInstructions(chatCtx: ChatContext) {\n  // loop in case there are items with the same id (shouldn't happen!)\n  while (true) {\n    const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n    if (idx !== undefined) {\n      chatCtx.items.splice(idx, 1);\n    } else {\n      break;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAA+B;AAE/B,0BAKO;AAEP,0BAMO;AACP,uBAA4C;AAC5C,iBAAoB;AACpB,gCAAkC;AAClC,mBAAiD;AACjD,mBAAkF;AAGlF,yBAA2B;AAIpB,MAAM,mBAAmB;AAAA,EAK9B,YACkB,YACA,gBAChB;AAFgB;AACA;AAEhB,SAAK,SAAK,wBAAU,OAAO;AAC3B,SAAK,qBAAqB,CAAC;AAAA,EAC7B;AAAA,EAVA,gBAAwB;AAAA,EACxB;AAAA,EACA;AASF;AAGO,MAAM,YAAY;AAAA,EACvB;AAAA,EACA;AAAA,EAEA,cAAc;AACZ,SAAK,SAAS,CAAC;AACf,SAAK,eAAe,IAAI,oBAAO;AAAA,EACjC;AACF;AAGO,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,UACA,gBACA,eACA,WACA;AACA,SAAK,WAAW;AAChB,SAAK,iBAAiB;AACtB,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAKX;AACD,UAAM,EAAE,UAAU,gBAAgB,gBAAgB,MAAM,UAAU,IAAI;AACtE,WAAO,IAAI,iBAAiB,UAAU,gBAAgB,eAAe,SAAS;AAAA,EAChF;AACF;AAEA,SAAS,kBAAkB,YAA8B;AACvD,QAAM,aAAa,CAAC,UAAU,UAAU,SAAS;AAEjD,MAAI,WAAW,SAAS,OAAO,UAAU,GAAG;AAC1C,WAAO;AAAA,EACT;AAEA,MAAI,eAAe,UAAa,eAAe,MAAM;AACnD,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,WAAO,WAAW,MAAM,iBAAiB;AAAA,EAC3C;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,UAAU,EAAE,MAAM,iBAAiB;AAAA,EACvD;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,WAAW,OAAO,CAAC,EAAE,MAAM,iBAAiB;AAAA,EAChE;AAEA,MAAI,sBAAsB,QAAQ;AAChC,WAAO,OAAO,QAAQ,UAAU,EAAE;AAAA,MAChC,CAAC,CAAC,KAAK,KAAK,MAAM,WAAW,SAAS,OAAO,GAAG,KAAK,kBAAkB,KAAK;AAAA,IAC9E;AAAA,EACF;AAEA,SAAO;AACT;AAEO,MAAM,oBAAoB;AAAA,EAC/B,YACkB,UACA,gBACA,WACA,WACA,cACA,eAChB;AANgB;AACA;AACA;AACA;AACA;AACA;AAAA,EACf;AAAA,EAEH,OAAO,OAAO,QAOX;AACD,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,IAClB,IAAI;AACJ,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAQO,MAAM,UAAU;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,cAAU,gBAAI;AAAA,EAEd,YAAY,UAAwB,QAAiB,WAA8B;AACjF,SAAK,WAAW;AAChB,SAAK,SAAS;AACd,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAAyE;AACrF,UAAM,EAAE,UAAU,SAAS,QAAW,YAAY,OAAU,IAAI;AAChE,WAAO,IAAI,UAAU,UAAU,QAAQ,SAAS;AAAA,EAClD;AAAA,EAEA,WAA6B;AAC3B,YAAI,iCAAY,KAAK,SAAS,GAAG;AAC/B,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ,KAAK,UAAU;AAAA,UACvB,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,YAAI,6BAAe,KAAK,SAAS,GAAG;AAClC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MACpD,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,cAAc,QAAW;AAChC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ;AAAA;AAAA,UACR,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,QAAI,YAA+B;AACnC,QAAI,aAAsB,KAAK;AAC/B,YAAI,oCAAe,KAAK,MAAM,GAAG;AAC/B,kBAAY,KAAK,OAAO;AACxB,mBAAa,KAAK,OAAO;AAAA,IAC3B;AAEA,QAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK,SAAS;AAAA,UACtB,UAAU,KAAK,SAAS;AAAA,QAC1B;AAAA,QACA,eAAe,KAAK,SAAS,IAAI;AAAA,MACnC;AACA,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB;AAAA,MAClB,CAAC;AAAA,IACH;AAEA,WAAO,iBAAiB,OAAO;AAAA,MAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MAClD,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,KAAK,SAAS;AAAA,QACpB,QAAQ,KAAK,SAAS;AAAA,QACtB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,QAChE,SAAS;AAAA,MACX,CAAC;AAAA,MACD,eAAe,eAAe;AAAA;AAAA,MAC9B;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,SAAS,iBAAiB,QAIT;AACtB,QAAM,EAAE,UAAU,QAAQ,UAAU,IAAI;AACxC,QAAM,aAAS,gBAAI;AAGnB,MAAI,cAAc;AAClB,MAAI,iBAAiB;AACrB,MAAI,kBAAkB,OAAO;AAC3B,qBAAiB;AACjB,kBAAc;AAAA,EAChB;AAEA,UAAI,iCAAY,cAAc,GAAG;AAC/B,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ,eAAe;AAAA,QACvB,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,UAAI,6BAAe,cAAc,GAAG;AAClC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,mBAAmB,QAAW;AAChC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA;AAAA,QACR,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,YAA+B;AACnC,MAAI,aAAsB;AAC1B,UAAI,oCAAe,WAAW,GAAG;AAC/B,gBAAY,YAAY;AACxB,iBAAa,YAAY;AAAA,EAC3B;AAEA,MAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAO;AAAA,MACL;AAAA,QACE,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA,MACV;AAAA,MACA,eAAe,SAAS,IAAI;AAAA,IAC9B;AACA,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,SAAO,oBAAoB,OAAO;AAAA,IAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,IAC7C,gBAAgB,uCAAmB,OAAO;AAAA,MACxC,MAAM,SAAS;AAAA,MACf,QAAQ,SAAS;AAAA,MACjB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,MAChE,SAAS;AAAA,IACX,CAAC;AAAA,IACD,eAAe,eAAe;AAAA;AAAA,IAC9B;AAAA,IACA,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACH;AAEA,MAAM,0BAA0B;AAazB,SAAS,mBAAmB,SAIhC;AACD,QAAM,EAAE,SAAS,cAAc,aAAa,IAAI;AAEhD,QAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,MAAI,QAAQ,QAAW;AACrB,QAAI,QAAQ,MAAM,GAAG,EAAG,SAAS,WAAW;AAE1C,cAAQ,MAAM,GAAG,IAAI,gCAAY,OAAO;AAAA,QACtC,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,WAAW,QAAQ,MAAM,GAAG,EAAG;AAAA,MACjC,CAAC;AAAA,IACH,OAAO;AACL,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAAA,EACF,WAAW,cAAc;AAEvB,YAAQ,MAAM;AAAA,MACZ,gCAAY,OAAO;AAAA,QACjB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,MACxB,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,SACA,SACA,eACA,YACkC;AAClC,QAAM,aAAa,IAAI,4CAA0B;AACjD,QAAM,iBAAiB,IAAI,4CAAgC;AAE3D,QAAM,aAAa,WAAW,SAAS,UAAU;AACjD,QAAM,iBAAiB,eAAe,SAAS,UAAU;AACzD,QAAM,OAAO,IAAI,mBAAmB,WAAW,UAAU,eAAe,QAAQ;AAEhF,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAA0E;AAC9E,QAAI,YAAuD;AAE3D,QAAI;AACF,kBAAY,MAAM,KAAK,SAAS,SAAS,aAAa;AACtD,UAAI,cAAc,MAAM;AACtB,cAAM,WAAW,MAAM;AACvB;AAAA,MACF;AAIA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AAEA,YAAI,OAAO,UAAU,UAAU;AAC7B,eAAK,iBAAiB;AACtB,gBAAM,WAAW,MAAM,KAAK;AAAA,QAE9B,OAAO;AACL,cAAI,MAAM,UAAU,QAAW;AAC7B;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,WAAW;AACzB,uBAAW,QAAQ,MAAM,MAAM,WAAW;AACxC,kBAAI,KAAK,SAAS,gBAAiB;AAEnC,oBAAM,WAAW,iCAAa,OAAO;AAAA,gBACnC,QAAQ,GAAG,KAAK,EAAE,QAAQ,KAAK,mBAAmB,MAAM;AAAA,gBACxD,MAAM,KAAK;AAAA,gBACX,MAAM,KAAK;AAAA,cACb,CAAC;AAED,mBAAK,mBAAmB,KAAK,QAAQ;AACrC,oBAAM,eAAe,MAAM,QAAQ;AAAA,YACrC;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,SAAS;AACvB,iBAAK,iBAAiB,MAAM,MAAM;AAClC,kBAAM,WAAW,MAAM,MAAM,MAAM,OAAO;AAAA,UAC5C;AAAA,QACF;AAAA,MAIF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,WAAW,MAAM;AACvB,YAAM,eAAe,MAAM;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,MACA,eACA,YAC0C;AAC1C,QAAM,cAAc,IAAI,4CAA8B;AACtD,QAAM,eAAe,YAAY,SAAS,UAAU;AACpD,QAAM,oBAAoB,YAAY;AAEtC,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAAkE;AACtE,QAAI,YAA+C;AAEnD,QAAI;AACF,kBAAY,MAAM,KAAK,MAAM,aAAa;AAC1C,UAAI,cAAc,MAAM;AACtB,cAAM,aAAa,MAAM;AACzB;AAAA,MACF;AAEA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AACA,cAAM,aAAa,MAAM,KAAK;AAAA,MAChC;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,aAAa,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAOA,eAAe,YACb,QACA,KACA,QACA,YACe;AACf,QAAM,SAAS,OAAO,UAAU;AAChC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,OAAO,SAAS;AAClB;AAAA,MACF;AACA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AACV,UAAI,QAAQ;AACZ,UAAI,eAAe,MAAM;AACvB,cAAM,WAAW,YAAY,KAAK;AAAA,MACpC;AACA,UAAI,CAAC,IAAI,aAAa,MAAM;AAC1B,YAAI,aAAa,QAAQ;AAAA,MAC3B;AAAA,IACF;AAAA,EACF,UAAE;AACA,QAAI,eAAe,MAAM;AACvB,iBAAW,MAAM;AAAA,IACnB;AACA,qCAAQ;AAAA,EACV;AACF;AAEO,SAAS,sBACd,QACA,YACA,YACwB;AACxB,QAAM,MAAM;AAAA,IACV,MAAM;AAAA,IACN,cAAc,IAAI,oBAAO;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,YAAY,QAAQ,KAAKA,YAAW,QAAQ,UAAU;AAAA,MACtE;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAOA,eAAe,aACb,WACA,YACA,KACA,QACe;AACf,QAAM,SAAS,UAAU,UAAU;AACnC,MAAI,YAAmC;AAEvC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,iCAAQ,SAAS;AACnB;AAAA,MACF;AAEA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AAEV,UAAI,MAAM,KAAK,KAAK;AAEpB,UACE,CAAC,IAAI,cAAc,QACnB,WAAW,cACX,WAAW,eAAe,MAAM,cAChC,CAAC,WACD;AACA,oBAAY,IAAI,+BAAe,MAAM,YAAY,WAAW,YAAY,CAAC;AAAA,MAC3E;AAEA,UAAI,WAAW;AACb,mBAAW,KAAK,UAAU,KAAK,KAAK,GAAG;AACrC,gBAAM,WAAW,aAAa,CAAC;AAAA,QACjC;AAAA,MACF,OAAO;AACL,cAAM,WAAW,aAAa,KAAK;AAAA,MACrC;AAIA,UAAI,CAAC,IAAI,cAAc,MAAM;AAC3B,YAAI,cAAc,QAAQ;AAAA,MAC5B;AAAA,IACF;AAAA,EACF,UAAE;AACA,qCAAQ;AACR,QAAI,WAAW;AACb,iBAAW,KAAK,UAAU,MAAM,GAAG;AACjC,cAAM,WAAW,aAAa,CAAC;AAAA,MACjC;AAAA,IACF;AACA,eAAW,MAAM;AAAA,EACnB;AACF;AAEO,SAAS,uBACd,WACA,aACA,YACyB;AACzB,QAAM,MAAM;AAAA,IACV,OAAO,CAAC;AAAA,IACR,eAAe,IAAI,oBAAO;AAAA,EAC5B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,aAAa,WAAW,aAAa,KAAKA,YAAW,MAAM;AAAA,MAC3E;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAyB,MAAM;AAAA,EAAC;AAAA,EAChC,2BAA2B,MAAM;AAAA,EAAC;AAAA,EAClC;AACF,GAS6B;AAC3B,QAAM,aAAS,gBAAI;AACnB,QAAM,aAAyB;AAAA,IAC7B,QAAQ,CAAC;AAAA,IACT,wBAAwB,IAAI,oBAAO;AAAA,EACrC;AAEA,QAAM,gBAAgB,CAAC,QAA6B;AAClD,6BAAyB,GAAG;AAC5B,eAAW,OAAO,KAAK,GAAG;AAAA,EAC5B;AAEA,QAAM,mBAAmB,OAAOA,gBAAgC;AAC9D,UAAM,SAASA,YAAW;AAC1B,UAAM,SAAS,eAAe,UAAU;AAExC,UAAM,QAAwB,CAAC;AAC/B,WAAO,CAAC,OAAO,SAAS;AACtB,YAAM,EAAE,MAAM,OAAO,SAAS,IAAI,MAAM,OAAO,KAAK;AACpD,UAAI,OAAO,QAAS;AACpB,UAAI,KAAM;AAEV,UAAI,eAAe,QAAQ;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA;AAAA,QACF;AACA;AAAA,MACF;AAIA,YAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,UAAI,CAAC,MAAM;AACT,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,uBAAuB,SAAS,IAAI;AAAA,QACtC;AACA;AAAA,MACF;AAEA,UAAI,KAAC,oCAAe,IAAI,GAAG;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,sBAAsB,OAAO,IAAI;AAAA,QACnC;AACA;AAAA,MACF;AAEA,UAAI;AAGJ,UAAI;AACF,cAAM,WAAW,KAAK,MAAM,SAAS,IAAI;AAEzC,gBAAI,8BAAY,KAAK,UAAU,GAAG;AAChC,gBAAM,SAAS,UAAM,iCAAuB,KAAK,YAAY,QAAQ;AACrE,cAAI,OAAO,SAAS;AAClB,yBAAa,OAAO;AAAA,UACtB,OAAO;AACL,kBAAM,OAAO;AAAA,UACf;AAAA,QACF,OAAO;AACL,uBAAa;AAAA,QACf;AAAA,MACF,SAAS,UAAU;AACjB,cAAM,YAAQ,sBAAQ,QAAQ;AAC9B,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,SAAS;AAAA,YACpB,WAAW,aAAa;AAAA,YACxB,OAAO,MAAM;AAAA,UACf;AAAA,UACA,6BAA6B,SAAS,IAAI;AAAA,QAC5C;AACA;AAAA,UACE,iBAAiB;AAAA,YACf;AAAA,YACA,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,UAAI,CAAC,WAAW,uBAAuB,MAAM;AAC3C,mBAAW,uBAAuB,QAAQ;AAAA,MAC5C;AAEA,6BAAuB,QAAQ;AAE/B,aAAO;AAAA,QACL;AAAA,UACE,UAAU,SAAS;AAAA,UACnB,WAAW;AAAA,UACX,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAEA,YAAM,gBAAgB,+BAAkB,IAAI,EAAE,cAAc,SAAS,GAAG,YAAY;AAClF,eAAO,MAAM,KAAK,QAAQ,YAAY;AAAA,UACpC,KAAK,IAAI,8BAAW,SAAS,cAAc,QAAQ;AAAA,UACnD,YAAY,SAAS;AAAA,UACrB,aAAa;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAED,YAAM,wBAAwB,OAAO,iBAAmC;AAItE,YAAIC;AACJ,YAAI;AACF,gBAAM,EAAE,QAAQ,UAAU,IAAI,MAAM,iBAAiB,cAAc,MAAM;AACzE,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,WAAW,YAAY,IAAI,MAAM,uBAAuB,IAAI;AAAA,YAC5D,QAAQ,YAAY,SAAY;AAAA,UAClC,CAAC;AAAA,QACH,SAAS,UAAU;AACjB,iBAAO;AAAA,YACL;AAAA,cACE,UAAU,SAAS;AAAA,cACnB,WAAW,aAAa;AAAA,cACxB,WAAO,sBAAQ,QAAQ,EAAE;AAAA,YAC3B;AAAA,YACA;AAAA,UACF;AACA,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,eAAW,sBAAQ,QAAQ;AAAA,UAC7B,CAAC;AAAA,QACH,UAAE;AACA,cAAI,CAACA,YAAY,OAAM,IAAI,MAAM,yBAAyB;AAC1D,wBAAcA,WAAU;AAAA,QAC1B;AAAA,MACF;AAGA,YAAM,KAAK,sBAAsB,aAAa,CAAC;AAAA,IACjD;AAEA,UAAM,QAAQ,WAAW,KAAK;AAC9B,QAAI,WAAW,OAAO,SAAS,GAAG;AAChC,aAAO;AAAA,QACL;AAAA,UACE,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO,CAAC,kBAAK,KAAK,kBAAkB,YAAY,uBAAuB,GAAG,UAAU;AACtF;AAYA,eAAe,iBAAoB,SAAqB,QAA0C;AAChG,QAAM,WAAW,IAAI,oBAAmB;AAExC,QAAM,eAAe,MAAM;AACzB,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,QAAW,WAAW,KAAK,CAAC;AAAA,IACzD;AAAA,EACF;AAEA,SAAO,iBAAiB,SAAS,YAAY;AAE7C,UACG,KAAK,CAAC,MAAM;AACX,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,GAAG,WAAW,MAAM,CAAC;AAAA,IAClD;AAAA,EACF,CAAC,EACA,MAAM,CAAC,MAAM;AACZ,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,OAAO,CAAC;AAAA,IACnB;AAAA,EACF,CAAC,EACA,QAAQ,MAAM;AACb,WAAO,oBAAoB,SAAS,YAAY;AAAA,EAClD,CAAC;AAEH,SAAO,MAAM,SAAS;AACxB;AAEO,SAAS,mBAAmB,SAAsB;AAEvD,SAAO,MAAM;AACX,UAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,QAAI,QAAQ,QAAW;AACrB,cAAQ,MAAM,OAAO,KAAK,CAAC;AAAA,IAC7B,OAAO;AACL;AAAA,IACF;AAAA,EACF;AACF;","names":["controller","toolOutput"]}