@livekit/agents 1.0.40 → 1.0.41

package/dist/voice/agent_activity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKpD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EAEtB,MAAM,wBAAwB,CAAC;AAqBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAclD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAA4C;IAE/D,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAErD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,sBAAsB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAE3B,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IA8F9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoH5B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBxD,aAAa,CAAC,EAAE,UAAU,EAAE,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAUvE,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAwB/D,gBAAgB,IAAI,IAAI;IAIxB,cAAc;IAUd,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAwDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IAmBxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAkBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAqCrD,eAAe,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQnC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAWtC,OAAO,CAAC,wBAAwB;IAyChC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAqB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IA+BxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA0C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IA4BlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAuCtB,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAuGhB,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC;IA0BzB,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAyHjB,OAAO;IAmIrB,OAAO,CAAC,sBAAsB,CAkZ5B;IAEF,OAAO,CAAC,iBAAiB,CA2BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;YAwc3B,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;IAiBhB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA2BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAyC7B"}
+{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKpD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EAEtB,MAAM,wBAAwB,CAAC;AAsBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAclD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAA4C;IAE/D,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAErD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,sBAAsB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAE3B,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IA8F9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoH5B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,uBAAuB,IAAI,OAAO,CAGrC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBxD,aAAa,CAAC,EAAE,UAAU,EAAE,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAUvE,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAwB/D,gBAAgB,IAAI,IAAI;IAIxB,cAAc;IAUd,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAwDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IAmBxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAkBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAqCrD,eAAe,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQnC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAWtC,OAAO,CAAC,wBAAwB;IAyChC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAqB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IA+BxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA0C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IA4BlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAuCtB,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAuGhB,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC;IA0BzB,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAyHjB,OAAO;IAmIrB,OAAO,CAAC,sBAAsB,CAqa5B;IAEF,OAAO,CAAC,iBAAiB,CA2BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;YAwc3B,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;IAiBhB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA2BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAyC7B"}

package/dist/voice/agent_activity.js

@@ -248,6 +248,9 @@ class AgentActivity {
   get allowInterruptions() {
     return this.agentSession.options.allowInterruptions;
   }
+  get useTtsAlignedTranscript() {
+    return this.agent.useTtsAlignedTranscript ?? this.agentSession.useTtsAlignedTranscript;
+  }
   get turnDetection() {
     return this.agentSession.turnDetection;
   }
@@ -893,7 +896,7 @@ ${instructions}` : instructions,
     } else {
       let audioOut = null;
       if (!audio) {
-        const [ttsTask, ttsStream] = performTTSInference(
+        const [ttsTask, ttsGenData] = performTTSInference(
           (...args) => this.agent.ttsNode(...args),
           audioSource,
           modelSettings,
@@ -901,7 +904,7 @@ ${instructions}` : instructions,
         );
         tasks.push(ttsTask);
         const [forwardTask, _audioOut] = performAudioForwarding(
-          ttsStream,
+          ttsGenData.audioStream,
           audioOutput,
           replyAbortController
         );
@@ -954,7 +957,7 @@ ${instructions}` : instructions,
     toolsMessages,
     span
   }) => {
-    var _a, _b, _c;
+    var _a, _b, _c, _d;
     speechHandle._agentTurnContext = otelContext.active();
     span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id);
     if (instructions) {
@@ -992,12 +995,12 @@ ${instructions}` : instructions,
     );
     tasks.push(llmTask);
     let ttsTask = null;
-    let ttsStream = null;
+    let ttsGenData = null;
     let llmOutput;
     if (audioOutput) {
       const [ttsTextInput, textOutput] = llmGenData.textStream.tee();
       llmOutput = textOutput;
-      [ttsTask, ttsStream] = performTTSInference(
+      [ttsTask, ttsGenData] = performTTSInference(
         (...args) => this.agent.ttsNode(...args),
         ttsTextInput,
         modelSettings,
@@ -1021,7 +1024,20 @@ ${instructions}` : instructions,
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForAuthorization()]);
     speechHandle._clearAuthorization();
     const replyStartedAt = Date.now();
-    const trNodeResult = await this.agent.transcriptionNode(llmOutput, modelSettings);
+    let transcriptionInput = llmOutput;
+    if (this.useTtsAlignedTranscript && ((_a = this.tts) == null ? void 0 : _a.capabilities.alignedTranscript) && ttsGenData) {
+      const timedTextsStream = await Promise.race([
+        ttsGenData.timedTextsFut.await,
+        (ttsTask == null ? void 0 : ttsTask.result.catch(
+          () => this.logger.warn("TTS task failed before resolving timedTextsFut")
+        )) ?? Promise.resolve()
+      ]);
+      if (timedTextsStream) {
+        this.logger.debug("Using TTS aligned transcripts for transcription node input");
+        transcriptionInput = timedTextsStream;
+      }
+    }
+    const trNodeResult = await this.agent.transcriptionNode(transcriptionInput, modelSettings);
     let textOut = null;
     if (trNodeResult) {
       const [textForwardTask, _textOut] = performTextForwarding(
@@ -1040,9 +1056,9 @@ ${instructions}` : instructions,
     };
     let audioOut = null;
     if (audioOutput) {
-      if (ttsStream) {
+      if (ttsGenData) {
         const [forwardTask, _audioOut] = performAudioForwarding(
-          ttsStream,
+          ttsGenData.audioStream,
           audioOutput,
           replyAbortController
         );
@@ -1050,7 +1066,7 @@ ${instructions}` : instructions,
         tasks.push(forwardTask);
         audioOut.firstFrameFut.await.then((ts) => onFirstFrame(ts)).catch(() => this.logger.debug("firstFrameFut cancelled before first frame"));
       } else {
-        throw Error("ttsStream is null when audioOutput is enabled");
+        throw Error("ttsGenData is null when audioOutput is enabled");
       }
     } else {
       textOut == null ? void 0 : textOut.firstTextFut.await.then(() => onFirstFrame()).catch(() => this.logger.debug("firstTextFut cancelled before first frame"));
@@ -1198,10 +1214,10 @@ ${instructions}` : instructions,
       this.logger.debug(
         {
           speechId: speechHandle.id,
-          name: (_a = sanitizedOut.toolCall) == null ? void 0 : _a.name,
+          name: (_b = sanitizedOut.toolCall) == null ? void 0 : _b.name,
           args: sanitizedOut.toolCall.args,
-          output: (_b = sanitizedOut.toolCallOutput) == null ? void 0 : _b.output,
-          isError: (_c = sanitizedOut.toolCallOutput) == null ? void 0 : _c.isError
+          output: (_c = sanitizedOut.toolCallOutput) == null ? void 0 : _c.output,
+          isError: (_d = sanitizedOut.toolCallOutput) == null ? void 0 : _d.isError
         },
         "Tool call execution finished"
       );
@@ -1368,14 +1384,14 @@ ${instructions}` : instructions,
           if (audioOutput) {
             let realtimeAudioResult = null;
             if (ttsTextInput) {
-              const [ttsTask, ttsStream] = performTTSInference(
+              const [ttsTask, ttsGenData] = performTTSInference(
                 (...args) => this.agent.ttsNode(...args),
                 ttsTextInput,
                 modelSettings,
                 abortController
               );
               tasks.push(ttsTask);
-              realtimeAudioResult = ttsStream;
+              realtimeAudioResult = ttsGenData.audioStream;
             } else if (msgModalities && msgModalities.includes("audio")) {
               realtimeAudioResult = await this.agent.realtimeAudioOutputNode(
                 msg.audioStream,