@livekit/agents 0.1.0 → 0.3.0

package/src/multimodal/multimodal_agent.ts

@@ -0,0 +1,428 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type {
+  LocalTrackPublication,
+  RemoteAudioTrack,
+  RemoteParticipant,
+  Room,
+} from '@livekit/rtc-node';
+import {
+  AudioSource,
+  AudioStream,
+  LocalAudioTrack,
+  RoomEvent,
+  TrackPublishOptions,
+  TrackSource,
+} from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import { AudioByteStream } from '../audio.js';
+import type * as llm from '../llm/index.js';
+import { log } from '../log.js';
+import { BasicTranscriptionForwarder } from '../transcription.js';
+import { findMicroTrackId } from '../utils.js';
+import { AgentPlayout, type PlayoutHandle } from './agent_playout.js';
+
+/**
+ * @internal
+ * @beta
+ */
+export abstract class RealtimeSession extends EventEmitter {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  abstract defaultConversation: any; // openai.realtime.Conversation
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  abstract inputAudioBuffer: any; // openai.realtime.InputAudioBuffer
+  abstract fncCtx: llm.FunctionContext | undefined;
+}
+
+/**
+ * @internal
+ * @beta
+ */
+export abstract class RealtimeModel {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  abstract session(options: any): RealtimeSession; // openai.realtime.ModelOptions
+  abstract close(): Promise<void>;
+  abstract sampleRate: number;
+  abstract numChannels: number;
+  abstract inFrameSize: number;
+  abstract outFrameSize: number;
+}
+
+export type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';
+export const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';
+
+/** @beta */
+export class MultimodalAgent {
+  model: RealtimeModel;
+  room: Room | null = null;
+  linkedParticipant: RemoteParticipant | null = null;
+  subscribedTrack: RemoteAudioTrack | null = null;
+  readMicroTask: { promise: Promise<void>; cancel: () => void } | null = null;
+
+  constructor({
+    model,
+    fncCtx,
+  }: {
+    model: RealtimeModel;
+    fncCtx?: llm.FunctionContext | undefined;
+  }) {
+    this.model = model;
+    this.#fncCtx = fncCtx;
+  }
+
+  #participant: RemoteParticipant | string | null = null;
+  #agentPublication: LocalTrackPublication | null = null;
+  #localTrackSid: string | null = null;
+  #localSource: AudioSource | null = null;
+  #agentPlayout: AgentPlayout | null = null;
+  #playingHandle: PlayoutHandle | undefined = undefined;
+  #logger = log();
+  #session: RealtimeSession | null = null;
+  #fncCtx: llm.FunctionContext | undefined = undefined;
+
+  #_started: boolean = false;
+  #_pendingFunctionCalls: Set<string> = new Set();
+  #_speaking: boolean = false;
+
+  get fncCtx(): llm.FunctionContext | undefined {
+    return this.#fncCtx!;
+  }
+
+  set fncCtx(ctx: llm.FunctionContext | undefined) {
+    this.#fncCtx = ctx;
+    if (this.#session) {
+      this.#session.fncCtx = ctx;
+    }
+  }
+
+  get #pendingFunctionCalls(): Set<string> {
+    return this.#_pendingFunctionCalls;
+  }
+
+  set #pendingFunctionCalls(calls: Set<string>) {
+    this.#_pendingFunctionCalls = calls;
+    this.#updateState();
+  }
+
+  get #speaking(): boolean {
+    return this.#_speaking;
+  }
+
+  set #speaking(isSpeaking: boolean) {
+    this.#_speaking = isSpeaking;
+    this.#updateState();
+  }
+
+  get #started(): boolean {
+    return this.#_started;
+  }
+
+  set #started(started: boolean) {
+    this.#_started = started;
+    this.#updateState();
+  }
+
+  start(
+    room: Room,
+    participant: RemoteParticipant | string | null = null,
+  ): Promise<RealtimeSession> {
+    return new Promise(async (resolve, reject) => {
+      if (this.#started) {
+        this.#logger.warn('MultimodalAgent already started');
+        resolve(this.#session!); // TODO: throw error?
+        return;
+      }
+      this.#updateState();
+
+      room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {
+        if (!this.linkedParticipant) {
+          return;
+        }
+
+        this.#linkParticipant(participant.identity);
+      });
+
+      this.room = room;
+      this.#participant = participant;
+
+      this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);
+      this.#agentPlayout = new AgentPlayout(
+        this.#localSource,
+        this.model.sampleRate,
+        this.model.numChannels,
+        this.model.inFrameSize,
+        this.model.outFrameSize,
+      );
+      const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);
+      const options = new TrackPublishOptions();
+      options.source = TrackSource.SOURCE_MICROPHONE;
+      this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;
+      if (!this.#agentPublication) {
+        this.#logger.error('Failed to publish track');
+        reject(new Error('Failed to publish track'));
+        return;
+      }
+
+      await this.#agentPublication.waitForSubscription();
+
+      if (participant) {
+        if (typeof participant === 'string') {
+          this.#linkParticipant(participant);
+        } else {
+          this.#linkParticipant(participant.identity);
+        }
+      } else {
+        // No participant specified, try to find the first participant in the room
+        for (const participant of room.remoteParticipants.values()) {
+          this.#linkParticipant(participant.identity);
+          break;
+        }
+      }
+
+      this.#session = this.model.session({ fncCtx: this.#fncCtx });
+      this.#started = true;
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('response_content_added', (message: any) => {
+        // openai.realtime.RealtimeContent
+        const trFwd = new BasicTranscriptionForwarder(
+          this.room!,
+          this.room!.localParticipant!.identity,
+          this.#getLocalTrackSid()!,
+          message.responseId,
+        );
+
+        const handle = this.#agentPlayout?.play(
+          message.itemId,
+          message.contentIndex,
+          trFwd,
+          message.textStream,
+          message.audioStream,
+        );
+        if (handle) {
+          this.#speaking = true;
+          handle.on('done', () => {
+            if (this.#playingHandle == handle) {
+              this.#speaking = false;
+            }
+          });
+        }
+        this.#playingHandle = handle;
+      });
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('input_speech_committed', (ev: any) => {
+        // openai.realtime.InputSpeechCommittedEvent
+        const participantIdentity = this.linkedParticipant?.identity;
+        const trackSid = this.subscribedTrack?.sid;
+        if (participantIdentity && trackSid) {
+          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);
+        } else {
+          this.#logger.error('Participant or track not set');
+        }
+      });
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('input_speech_transcription_completed', (ev: any) => {
+        // openai.realtime.InputSpeechTranscriptionCompletedEvent
+        const transcription = ev.transcript;
+        const participantIdentity = this.linkedParticipant?.identity;
+        const trackSid = this.subscribedTrack?.sid;
+        if (participantIdentity && trackSid) {
+          this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);
+        } else {
+          this.#logger.error('Participant or track not set');
+        }
+      });
+
+      this.#session.on('input_speech_started', (ev: any) => {
+        if (this.#playingHandle && !this.#playingHandle.done) {
+          this.#playingHandle.interrupt();
+
+          this.#session!.defaultConversation.item.truncate(
+            this.#playingHandle.itemId,
+            this.#playingHandle.contentIndex,
+            Math.floor((this.#playingHandle.audioSamples / 24000) * 1000),
+          );
+
+          this.#playingHandle = undefined;
+        }
+
+        const participantIdentity = this.linkedParticipant?.identity;
+        const trackSid = this.subscribedTrack?.sid;
+        if (participantIdentity && trackSid) {
+          this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);
+        }
+      });
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('function_call_started', (ev: any) => {
+        this.#pendingFunctionCalls.add(ev.callId);
+        this.#updateState();
+      });
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('function_call_completed', (ev: any) => {
+        this.#pendingFunctionCalls.delete(ev.callId);
+        this.#updateState();
+      });
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      this.#session.on('function_call_failed', (ev: any) => {
+        this.#pendingFunctionCalls.delete(ev.callId);
+        this.#updateState();
+      });
+
+      resolve(this.#session!);
+    });
+  }
+
+  // TODO
+  // close() {
+  //   if (!this.connected || !this.ws) return;
+  //   this.logger.debug('stopping assistant');
+  //   this.ws.close();
+  // }
+
+  #linkParticipant(participantIdentity: string): void {
+    if (!this.room) {
+      this.#logger.error('Room is not set');
+      return;
+    }
+
+    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;
+    if (!this.linkedParticipant) {
+      this.#logger.error(`Participant with identity ${participantIdentity} not found`);
+      return;
+    }
+
+    if (this.linkedParticipant.trackPublications.size > 0) {
+      this.#subscribeToMicrophone();
+    } else {
+      this.room.on(RoomEvent.TrackPublished, () => {
+        this.#subscribeToMicrophone();
+      });
+    }
+  }
+
+  #subscribeToMicrophone(): void {
+    const readAudioStreamTask = async (audioStream: AudioStream) => {
+      const bstream = new AudioByteStream(
+        this.model.sampleRate,
+        this.model.numChannels,
+        this.model.inFrameSize,
+      );
+
+      for await (const frame of audioStream) {
+        const audioData = frame.data;
+        for (const frame of bstream.write(audioData.buffer)) {
+          this.#session!.inputAudioBuffer.append(frame);
+        }
+      }
+    };
+
+    if (!this.linkedParticipant) {
+      this.#logger.error('Participant is not set');
+      return;
+    }
+
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+      if (publication.source !== TrackSource.SOURCE_MICROPHONE) {
+        continue;
+      }
+
+      if (!publication.subscribed) {
+        publication.setSubscribed(true);
+      }
+
+      const track = publication.track;
+
+      if (track && track !== this.subscribedTrack) {
+        this.subscribedTrack = track!;
+
+        if (this.readMicroTask) {
+          this.readMicroTask.cancel();
+        }
+
+        let cancel: () => void;
+        this.readMicroTask = {
+          promise: new Promise<void>((resolve, reject) => {
+            cancel = () => {
+              reject(new Error('Task cancelled'));
+            };
+            readAudioStreamTask(
+              new AudioStream(track, this.model.sampleRate, this.model.numChannels),
+            )
+              .then(resolve)
+              .catch(reject);
+          }),
+          cancel: () => cancel(),
+        };
+      }
+    }
+  }
+
+  #getLocalTrackSid(): string | null {
+    if (!this.#localTrackSid && this.room && this.room.localParticipant) {
+      this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);
+    }
+    return this.#localTrackSid;
+  }
+
+  #publishTranscription(
+    participantIdentity: string,
+    trackSid: string,
+    text: string,
+    isFinal: boolean,
+    id: string,
+  ): void {
+    this.#logger.debug(
+      `Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`,
+    );
+    if (!this.room?.localParticipant) {
+      this.#logger.error('Room or local participant not set');
+      return;
+    }
+
+    this.room.localParticipant.publishTranscription({
+      participantIdentity,
+      trackSid,
+      segments: [
+        {
+          text,
+          final: isFinal,
+          id,
+          startTime: BigInt(0),
+          endTime: BigInt(0),
+          language: '',
+        },
+      ],
+    });
+  }
+
+  #updateState() {
+    let newState: AgentState = 'initializing';
+    if (this.#pendingFunctionCalls.size > 0) {
+      newState = 'thinking';
+    } else if (this.#speaking) {
+      newState = 'speaking';
+    } else if (this.#started) {
+      newState = 'listening';
+    }
+
+    this.#setState(newState);
+  }
+
+  #setState(state: AgentState) {
+    if (this.room?.isConnected && this.room.localParticipant) {
+      const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
+      if (currentState !== state) {
+        this.room.localParticipant!.setAttributes({
+          [AGENT_STATE_ATTRIBUTE]: state,
+        });
+        this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);
+      }
+    }
+  }
+}

package/src/stt/index.ts

@@ -2,5 +2,5 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-export { STT, SpeechEvent, SpeechEventType, SpeechStream } from './stt.js';
+export { STT, SpeechEvent, SpeechEventType, SpeechStream, type SpeechData } from './stt.js';
 export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';

package/src/stt/stream_adapter.ts

@@ -7,24 +7,24 @@ import { VADEventType, type VADStream } from '../vad.js';
 import { STT, SpeechEvent, SpeechEventType, SpeechStream } from './stt.js';
 
 export class StreamAdapterWrapper extends SpeechStream {
-  closed: boolean;
-  stt: STT;
-  vadStream: VADStream;
-  eventQueue: (SpeechEvent | undefined)[];
-  language?: string;
-  task: {
+  #closed: boolean;
+  #stt: STT;
+  #vadStream: VADStream;
+  #eventQueue: (SpeechEvent | undefined)[];
+  #language?: string;
+  #task: {
     run: Promise<void>;
     cancel: () => void;
   };
 
   constructor(stt: STT, vadStream: VADStream, language: string | undefined = undefined) {
     super();
-    this.closed = false;
-    this.stt = stt;
-    this.vadStream = vadStream;
-    this.eventQueue = [];
-    this.language = language;
-    this.task = {
+    this.#closed = false;
+    this.#stt = stt;
+    this.#vadStream = vadStream;
+    this.#eventQueue = [];
+    this.#language = language;
+    this.#task = {
       run: new Promise((_, reject) => {
         this.run(reject);
       }),
@@ -33,46 +33,46 @@ export class StreamAdapterWrapper extends SpeechStream {
   }
 
   async run(reject: (arg: Error) => void) {
-    this.task.cancel = () => {
-      this.closed = true;
+    this.#task.cancel = () => {
+      this.#closed = true;
       reject(new Error('cancelled'));
     };
 
-    for (const event of this.vadStream) {
+    for (const event of this.#vadStream) {
       if (event.type == VADEventType.START_OF_SPEECH) {
         const startEvent = new SpeechEvent(SpeechEventType.START_OF_SPEECH);
-        this.eventQueue.push(startEvent);
+        this.#eventQueue.push(startEvent);
       } else if (event.type == VADEventType.END_OF_SPEECH) {
         const mergedFrames = mergeFrames(event.speech);
-        const endEvent = await this.stt.recognize(mergedFrames, this.language);
-        this.eventQueue.push(endEvent);
+        const endEvent = await this.#stt.recognize(mergedFrames, this.#language);
+        this.#eventQueue.push(endEvent);
       }
     }
 
-    this.eventQueue.push(undefined);
+    this.#eventQueue.push(undefined);
   }
 
   pushFrame(frame: AudioFrame) {
-    if (this.closed) {
+    if (this.#closed) {
       throw new TypeError('cannot push frame to closed stream');
     }
 
-    this.vadStream.pushFrame(frame);
+    this.#vadStream.pushFrame(frame);
   }
 
   async close(wait: boolean = true): Promise<void> {
-    this.closed = true;
+    this.#closed = true;
 
     if (!wait) {
-      this.task.cancel();
+      this.#task.cancel();
     }
 
-    await this.vadStream.close(wait);
-    await this.task.run;
+    await this.#vadStream.close(wait);
+    await this.#task.run;
   }
 
   next(): IteratorResult<SpeechEvent> {
-    const item = this.eventQueue.shift();
+    const item = this.#eventQueue.shift();
     if (item) {
       return { done: false, value: item };
     } else {
@@ -82,23 +82,23 @@ export class StreamAdapterWrapper extends SpeechStream {
 }
 
 export class StreamAdapter extends STT {
-  stt: STT;
-  vadStream: VADStream;
+  #stt: STT;
+  #vadStream: VADStream;
 
   constructor(stt: STT, vadStream: VADStream) {
     super(true);
-    this.stt = stt;
-    this.vadStream = vadStream;
+    this.#stt = stt;
+    this.#vadStream = vadStream;
   }
 
   async recognize(
     buffer: AudioBuffer,
     language: string | undefined = undefined,
   ): Promise<SpeechEvent> {
-    return await this.stt.recognize(buffer, language);
+    return await this.#stt.recognize(buffer, language);
   }
 
   stream(language: string | undefined = undefined) {
-    return new StreamAdapterWrapper(this.stt, this.vadStream, language);
+    return new StreamAdapterWrapper(this.#stt, this.#vadStream, language);
   }
 }

package/src/stt/stt.ts

@@ -5,9 +5,25 @@ import type { AudioFrame } from '@livekit/rtc-node';
 import type { AudioBuffer } from '../utils.js';
 
 export enum SpeechEventType {
+  /**
+   * Indicate the start of speech.
+   * If the STT doesn't support this event, this will be emitted at the same time
+   * as the first INTERMIN_TRANSCRIPT.
+   */
   START_OF_SPEECH = 0,
+  /**
+   * Interim transcript, useful for real-time transcription.
+   */
   INTERIM_TRANSCRIPT = 1,
+  /**
+   * Final transcript, emitted when the STT is confident enough that a certain
+   * portion of the speech will not change.
+   */
   FINAL_TRANSCRIPT = 2,
+  /**
+   * Indicate the end of speech, emitted when the user stops speaking.
+   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.
+   */
   END_OF_SPEECH = 3,
 }
 
@@ -30,8 +46,19 @@ export class SpeechEvent {
 }
 
 export abstract class SpeechStream implements IterableIterator<SpeechEvent> {
+  /**
+   * Push a frame to be recognised.
+   * It is recommended to push frames as soon as they are available.
+   */
   abstract pushFrame(token: AudioFrame): void;
 
+  /**
+   * Close the stream.
+   *
+   * @param wait
+   *   Whether to wait for the STT to finish processing the remaining
+   *   frames before closing
+   */
   abstract close(wait: boolean): Promise<void>;
 
   abstract next(): IteratorResult<SpeechEvent>;