@livekit/agents 0.7.3 → 0.7.5

package/src/pipeline/agent_playout.ts

@@ -168,15 +168,9 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
           handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;
           handle.synchronizer.pushAudio(frame);
           await this.#audioSource.captureFrame(frame);
-          await this.#audioSource.waitForPlayout();
         }
 
-        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,
-        //            but for some reason too many TTS frames can gunk up the buffer and lead to
-        //            FFI errors. this works 🤷‍♀️
-        // if (this.#audioSource.queuedDuration > 0) {
-        //   await this.#audioSource.waitForPlayout();
-        // }
+        await this.#audioSource.waitForPlayout();
 
         handle.synchronizer.close(false);
         resolve();

package/src/pipeline/pipeline_agent.ts

@@ -17,6 +17,11 @@ import {
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import { randomUUID } from 'node:crypto';
 import EventEmitter from 'node:events';
+import {
+  ATTRIBUTE_TRANSCRIPTION_FINAL,
+  ATTRIBUTE_TRANSCRIPTION_TRACK_ID,
+  TOPIC_TRANSCRIPTION,
+} from '../constants.js';
 import type {
   CallableFunctionResult,
   FunctionCallInfo,
@@ -518,28 +523,21 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       this.emit(VPAEvent.USER_STOPPED_SPEAKING);
       this.#deferredValidation.onHumanEndOfSpeech(event);
     });
-    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, (event) => {
+    this.#humanInput.on(HumanInputEvent.INTERIM_TRANSCRIPT, async (event) => {
       if (!this.#transcriptionId) {
         this.#transcriptionId = randomUUID();
       }
       this.#transcribedInterimText = event.alternatives![0].text;
 
-      this.#room!.localParticipant!.publishTranscription({
-        participantIdentity: this.#humanInput!.participant.identity,
-        trackSid: this.#humanInput!.subscribedTrack!.sid!,
-        segments: [
-          {
-            text: this.#transcribedInterimText,
-            id: this.#transcriptionId,
-            final: true,
-            startTime: BigInt(0),
-            endTime: BigInt(0),
-            language: '',
-          },
-        ],
-      });
+      await this.#publishTranscription(
+        this.#humanInput!.participant.identity,
+        this.#humanInput!.subscribedTrack!.sid!,
+        this.#transcribedInterimText,
+        false,
+        this.#transcriptionId,
+      );
     });
-    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, (event) => {
+    this.#humanInput.on(HumanInputEvent.FINAL_TRANSCRIPT, async (event) => {
       const newTranscript = event.alternatives![0].text;
       if (!newTranscript) return;
 
@@ -550,20 +548,14 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
       this.#lastFinalTranscriptTime = Date.now();
       this.transcribedText += (this.transcribedText ? ' ' : '') + newTranscript;
 
-      this.#room!.localParticipant!.publishTranscription({
-        participantIdentity: this.#humanInput!.participant.identity,
-        trackSid: this.#humanInput!.subscribedTrack!.sid!,
-        segments: [
-          {
-            text: this.transcribedText,
-            id: this.#transcriptionId,
-            final: true,
-            startTime: BigInt(0),
-            endTime: BigInt(0),
-            language: '',
-          },
-        ],
-      });
+      await this.#publishTranscription(
+        this.#humanInput!.participant.identity,
+        this.#humanInput!.subscribedTrack!.sid!,
+        this.transcribedText,
+        true,
+        this.#transcriptionId,
+      );
+
       this.#transcriptionId = undefined;
 
       if (
@@ -894,18 +886,54 @@ export class VoicePipelineAgent extends (EventEmitter as new () => TypedEmitter<
     handle.setDone();
   }
 
+  async #publishTranscription(
+    participantIdentity: string,
+    trackSid: string,
+    text: string,
+    isFinal: boolean,
+    id: string,
+  ) {
+    this.#room!.localParticipant!.publishTranscription({
+      participantIdentity: participantIdentity,
+      trackSid: trackSid,
+      segments: [
+        {
+          text: text,
+          final: isFinal,
+          id: id,
+          startTime: BigInt(0),
+          endTime: BigInt(0),
+          language: '',
+        },
+      ],
+    });
+    const stream = await this.#room!.localParticipant!.streamText({
+      senderIdentity: participantIdentity,
+      topic: TOPIC_TRANSCRIPTION,
+      attributes: {
+        [ATTRIBUTE_TRANSCRIPTION_TRACK_ID]: trackSid,
+        [ATTRIBUTE_TRANSCRIPTION_FINAL]: isFinal.toString(),
+      },
+    });
+    await stream.write(text);
+    await stream.close();
+  }
+
   #synthesizeAgentSpeech(
     speechId: string,
     source: string | LLMStream | AsyncIterable<string>,
   ): SynthesisHandle {
     const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
-    synchronizer.on('textUpdated', (text) => {
+    // TODO: where possible we would want to use deltas instead of full text segments, esp for LLM streams over the streamText API
+    synchronizer.on('textUpdated', async (text) => {
       this.#agentTranscribedText = text.text;
-      this.#room!.localParticipant!.publishTranscription({
-        participantIdentity: this.#room!.localParticipant!.identity,
-        trackSid: this.#agentPublication!.sid!,
-        segments: [text],
-      });
+      await this.#publishTranscription(
+        this.#room!.localParticipant!.identity!,
+        this.#agentPublication?.sid ?? '',
+        text.text,
+        text.final,
+        text.id,
+      );
     });
 
     if (!this.#agentOutput) {