@livekit/agents 0.6.3 → 0.7.0

package/src/ipc/{proc_job_executor.ts → supervised_proc.ts}

@@ -6,31 +6,54 @@ import { once } from 'node:events';
 import type { RunningJobInfo } from '../job.js';
 import { log, loggerOptions } from '../log.js';
 import { Future } from '../utils.js';
-import type { ProcOpts } from './job_executor.js';
-import { JobExecutor } from './job_executor.js';
 import type { IPCMessage } from './message.js';
 
-export class ProcJobExecutor extends JobExecutor {
+export interface ProcOpts {
+  initializeTimeout: number;
+  closeTimeout: number;
+  memoryWarnMB: number;
+  memoryLimitMB: number;
+  pingInterval: number;
+  pingTimeout: number;
+  highPingThreshold: number;
+}
+
+export abstract class SupervisedProc {
   #opts: ProcOpts;
   #started = false;
   #closing = false;
   #runningJob?: RunningJobInfo = undefined;
-  #proc?: ChildProcess;
+  proc?: ChildProcess;
   #pingInterval?: ReturnType<typeof setInterval>;
+  #memoryWatch?: ReturnType<typeof setInterval>;
   #pongTimeout?: ReturnType<typeof setTimeout>;
-  #init = new Future();
+  protected init = new Future();
   #join = new Future();
   #logger = log().child({ runningJob: this.#runningJob });
 
-  constructor(agent: string, initializeTimeout: number, closeTimeout: number) {
-    super();
+  constructor(
+    initializeTimeout: number,
+    closeTimeout: number,
+    memoryWarnMB: number,
+    memoryLimitMB: number,
+    pingInterval: number,
+    pingTimeout: number,
+    highPingThreshold: number,
+  ) {
     this.#opts = {
-      agent,
       initializeTimeout,
       closeTimeout,
+      memoryWarnMB,
+      memoryLimitMB,
+      pingInterval,
+      pingTimeout,
+      highPingThreshold,
     };
   }
 
+  abstract createProcess(): ChildProcess;
+  abstract mainTask(child: ChildProcess): Promise<void>;
+
   get started(): boolean {
     return this.#started;
   }
@@ -46,36 +69,50 @@ export class ProcJobExecutor extends JobExecutor {
       throw new Error('runner is closed');
     }
 
-    this.#proc = await import('./job_main.js').then((m) =>
-      m.runProcess({
-        agentFile: this.#opts.agent,
-      }),
-    );
+    this.proc = this.createProcess();
 
     this.#started = true;
     this.run();
   }
 
   async run() {
-    await this.#init.await;
+    await this.init.await;
 
     this.#pingInterval = setInterval(() => {
-      this.#proc!.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
-    }, this.PING_INTERVAL);
+      this.proc!.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
+    }, this.#opts.pingInterval);
 
     this.#pongTimeout = setTimeout(() => {
       this.#logger.warn('job is unresponsive');
       clearTimeout(this.#pongTimeout);
       clearInterval(this.#pingInterval);
-      this.#proc!.kill();
+      this.proc!.kill();
       this.#join.resolve();
-    }, this.PING_TIMEOUT);
+    }, this.#opts.pingTimeout);
+
+    this.#memoryWatch = setInterval(() => {
+      const memoryMB = process.memoryUsage().heapUsed / (1024 * 1024);
+      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {
+        this.#logger
+          .child({ memoryUsageMB: memoryMB, memoryLimitMB: this.#opts.memoryLimitMB })
+          .error('process exceeded memory limit, killing process');
+        this.close();
+      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {
+        this.#logger
+          .child({
+            memoryUsageMB: memoryMB,
+            memoryWarnMB: this.#opts.memoryWarnMB,
+            memoryLimitMB: this.#opts.memoryLimitMB,
+          })
+          .error('process memory usage is high');
+      }
+    });
 
     const listener = (msg: IPCMessage) => {
       switch (msg.case) {
         case 'pongResponse': {
           const delay = Date.now() - msg.value.timestamp;
-          if (delay > this.HIGH_PING_THRESHOLD) {
+          if (delay > this.#opts.highPingThreshold) {
             this.#logger.child({ delay }).warn('job executor is unresponsive');
           }
           this.#pongTimeout?.refresh();
@@ -87,21 +124,26 @@ export class ProcJobExecutor extends JobExecutor {
         }
         case 'done': {
           this.#closing = true;
-          this.#proc!.off('message', listener);
+          this.proc!.off('message', listener);
           this.#join.resolve();
           break;
         }
       }
     };
-    this.#proc!.on('message', listener);
-    this.#proc!.on('error', (err) => {
+    this.proc!.on('message', listener);
+    this.proc!.on('error', (err) => {
       if (this.#closing) return;
-      this.#logger.child({ err }).warn('job process exited unexpectedly');
+      this.#logger
+        .child({ err })
+        .warn('job process exited unexpectedly; this likely means the error above caused a crash');
       clearTimeout(this.#pongTimeout);
       clearInterval(this.#pingInterval);
+      clearInterval(this.#memoryWatch);
       this.#join.resolve();
     });
 
+    this.mainTask(this.proc!);
+
     await this.#join.await;
   }
 
@@ -116,17 +158,25 @@ export class ProcJobExecutor extends JobExecutor {
   async initialize() {
     const timer = setTimeout(() => {
       const err = new Error('runner initialization timed out');
-      this.#init.reject(err);
+      this.init.reject(err);
       throw err;
     }, this.#opts.initializeTimeout);
-    this.#proc!.send({ case: 'initializeRequest', value: { loggerOptions } });
-    await once(this.#proc!, 'message').then(([msg]: IPCMessage[]) => {
+    this.proc!.send({
+      case: 'initializeRequest',
+      value: {
+        loggerOptions,
+        pingInterval: this.#opts.pingInterval,
+        pingTimeout: this.#opts.pingTimeout,
+        highPingThreshold: this.#opts.highPingThreshold,
+      },
+    });
+    await once(this.proc!, 'message').then(([msg]: IPCMessage[]) => {
       clearTimeout(timer);
       if (msg!.case !== 'initializeResponse') {
         throw new Error('first message must be InitializeResponse');
       }
     });
-    this.#init.resolve();
+    this.init.resolve();
   }
 
   async close() {
@@ -136,11 +186,11 @@ export class ProcJobExecutor extends JobExecutor {
     this.#closing = true;
 
     if (!this.#runningJob) {
-      this.#proc!.kill();
+      this.proc!.kill();
       this.#join.resolve();
     }
 
-    this.#proc!.send({ case: 'shutdownRequest' });
+    this.proc!.send({ case: 'shutdownRequest' });
 
     const timer = setTimeout(() => {
       this.#logger.error('job shutdown is taking too much time');
@@ -157,6 +207,6 @@ export class ProcJobExecutor extends JobExecutor {
       throw new Error('executor already has a running job');
     }
     this.#runningJob = info;
-    this.#proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
+    this.proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
   }
 }

package/src/job.ts

@@ -11,8 +11,21 @@ import type {
 } from '@livekit/rtc-node';
 import { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';
 import type { Logger } from 'pino';
+import type { InferenceExecutor } from './ipc/inference_executor.js';
 import { log } from './log.js';
 
+export class CurrentJobContext {
+  static #current: JobContext;
+
+  constructor(proc: JobContext) {
+    CurrentJobContext.#current = proc;
+  }
+
+  static getCurrent(): JobContext {
+    return CurrentJobContext.#current;
+  }
+}
+
 /** Which tracks, if any, should the agent automatically subscribe to? */
 export enum AutoSubscribe {
   SUBSCRIBE_ALL,
@@ -60,6 +73,7 @@ export class JobContext {
     };
   } = {};
   #logger: Logger;
+  #inferenceExecutor: InferenceExecutor;
 
   constructor(
     proc: JobProcess,
@@ -67,6 +81,7 @@ export class JobContext {
     room: Room,
     onConnect: () => void,
     onShutdown: (s: string) => void,
+    inferenceExecutor: InferenceExecutor,
   ) {
     this.#proc = proc;
     this.#info = info;
@@ -76,6 +91,7 @@ export class JobContext {
     this.onParticipantConnected = this.onParticipantConnected.bind(this);
     this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
     this.#logger = log().child({ info: this.#info });
+    this.#inferenceExecutor = inferenceExecutor;
   }
 
   get proc(): JobProcess {
@@ -96,6 +112,11 @@ export class JobContext {
     return this.#room.localParticipant;
   }
 
+  /** @returns The global inference executor */
+  get inferenceExecutor(): InferenceExecutor {
+    return this.#inferenceExecutor;
+  }
+
   /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */
   addShutdownCallback(callback: () => Promise<void>) {
     this.shutdownCallbacks.push(callback);

package/src/metrics/base.ts

@@ -86,16 +86,13 @@ export class MultimodalLLMError extends Error {
   type?: string;
   reason?: string;
   code?: string;
-  constructor(
-    {
-      type,
-      reason,
-      code,
-      message,
-    }: { type?: string; reason?: string; code?: string; message?: string } = {},
-    options?: ErrorOptions,
-  ) {
-    super(message, options);
+  constructor({
+    type,
+    reason,
+    code,
+    message,
+  }: { type?: string; reason?: string; code?: string; message?: string } = {}) {
+    super(message);
     this.type = type;
     this.reason = reason;
     this.code = code;

package/src/multimodal/agent_playout.ts

@@ -5,7 +5,7 @@ import type { AudioFrame } from '@livekit/rtc-node';
 import { type AudioSource } from '@livekit/rtc-node';
 import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
-import type { TranscriptionForwarder } from '../transcription.js';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
 
 export const proto = {};
@@ -16,7 +16,7 @@ export class PlayoutHandle extends EventEmitter {
   #itemId: string;
   #contentIndex: number;
   /** @internal */
-  transcriptionFwd: TranscriptionForwarder;
+  synchronizer: TextAudioSynchronizer;
   /** @internal */
   doneFut: Future;
   /** @internal */
@@ -33,14 +33,14 @@ export class PlayoutHandle extends EventEmitter {
     sampleRate: number,
     itemId: string,
     contentIndex: number,
-    transcriptionFwd: TranscriptionForwarder,
+    synchronizer: TextAudioSynchronizer,
   ) {
     super();
     this.#audioSource = audioSource;
     this.#sampleRate = sampleRate;
     this.#itemId = itemId;
     this.#contentIndex = contentIndex;
-    this.transcriptionFwd = transcriptionFwd;
+    this.synchronizer = synchronizer;
     this.doneFut = new Future();
     this.intFut = new Future();
     this.#interrupted = false;
@@ -63,7 +63,7 @@ export class PlayoutHandle extends EventEmitter {
   }
 
   get textChars(): number {
-    return this.transcriptionFwd.currentCharacterIndex;
+    return this.synchronizer.playedText.length;
   }
 
   get contentIndex(): number {
@@ -111,7 +111,7 @@ export class AgentPlayout extends EventEmitter {
   play(
     itemId: string,
     contentIndex: number,
-    transcriptionFwd: TranscriptionForwarder,
+    synchronizer: TextAudioSynchronizer,
     textStream: AsyncIterableQueue<string>,
     audioStream: AsyncIterableQueue<AudioFrame>,
   ): PlayoutHandle {
@@ -120,7 +120,7 @@ export class AgentPlayout extends EventEmitter {
       this.#sampleRate,
       itemId,
       contentIndex,
-      transcriptionFwd,
+      synchronizer,
     );
     this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
     return handle;
@@ -159,8 +159,9 @@ export class AgentPlayout extends EventEmitter {
                     if (cancelledText || cancelled) {
                       break;
                     }
-                    handle.transcriptionFwd.pushText(text);
+                    handle.synchronizer.pushText(text);
                   }
+                  handle.synchronizer.markTextSegmentEnd();
                   resolveText();
                 } catch (error) {
                   rejectText(error);
@@ -189,12 +190,12 @@ export class AgentPlayout extends EventEmitter {
                       break;
                     }
                     if (firstFrame) {
-                      handle.transcriptionFwd.start();
+                      handle.synchronizer.segmentPlayoutStarted();
                       this.emit('playout_started');
                       firstFrame = false;
                     }
 
-                    handle.transcriptionFwd.pushAudio(frame);
+                    handle.synchronizer.pushAudio(frame);
 
                     for (const f of bstream.write(frame.data.buffer)) {
                       handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;
@@ -208,7 +209,7 @@ export class AgentPlayout extends EventEmitter {
                       await this.#audioSource.captureFrame(f);
                     }
 
-                    handle.transcriptionFwd.markAudioComplete();
+                    handle.synchronizer.markAudioSegmentEnd();
 
                     await this.#audioSource.waitForPlayout();
                   }
@@ -233,6 +234,7 @@ export class AgentPlayout extends EventEmitter {
             handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
 
             if (handle.interrupted || captureTask.error) {
+              await handle.synchronizer.close(true);
               this.#audioSource.clearQueue(); // make sure to remove any queued frames
             }
 
@@ -241,15 +243,11 @@ export class AgentPlayout extends EventEmitter {
             }
 
             if (!firstFrame) {
-              if (!handle.interrupted) {
-                handle.transcriptionFwd.markTextComplete();
-              }
-
               this.emit('playout_stopped', handle.interrupted);
             }
 
             handle.doneFut.resolve();
-            await handle.transcriptionFwd.close(handle.interrupted);
+            await handle.synchronizer.close(false);
           }
 
           resolve();

package/src/multimodal/multimodal_agent.ts

@@ -22,7 +22,7 @@ import { AudioByteStream } from '../audio.js';
 import * as llm from '../llm/index.js';
 import { log } from '../log.js';
 import type { MultimodalLLMMetrics } from '../metrics/base.js';
-import { BasicTranscriptionForwarder } from '../transcription.js';
+import { TextAudioSynchronizer, defaultTextSyncOptions } from '../transcription.js';
 import { findMicroTrackId } from '../utils.js';
 import { AgentPlayout, type PlayoutHandle } from './agent_playout.js';
 
@@ -190,7 +190,7 @@ export class MultimodalAgent extends EventEmitter {
         this.emit('agent_stopped_speaking');
         this.#speaking = false;
         if (this.#playingHandle) {
-          let text = this.#playingHandle.transcriptionFwd.text;
+          let text = this.#playingHandle.synchronizer.playedText;
           if (interrupted) {
             text += '…';
           }
@@ -245,17 +245,21 @@ export class MultimodalAgent extends EventEmitter {
         // openai.realtime.RealtimeContent
         if (message.contentType === 'text') return;
 
-        const trFwd = new BasicTranscriptionForwarder(
-          this.room!,
-          this.room!.localParticipant!.identity!,
-          this.#getLocalTrackSid()!,
-          message.responseId,
-        );
+        const synchronizer = new TextAudioSynchronizer(defaultTextSyncOptions);
+        synchronizer.on('textUpdated', (text) => {
+          this.#publishTranscription(
+            this.room!.localParticipant!.identity!,
+            this.#getLocalTrackSid()!,
+            text.text,
+            text.final,
+            text.id,
+          );
+        });
 
         const handle = this.#agentPlayout?.play(
           message.itemId,
           message.contentIndex,
-          trFwd,
+          synchronizer,
           message.textStream,
           message.audioStream,
         );

package/src/pipeline/agent_output.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import { log } from '../log.js';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { SynthesizeStream, type TTS } from '../tts/index.js';
 import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
 import type { AgentPlayout, PlayoutHandle } from './agent_playout.js';
@@ -21,12 +22,20 @@ export class SynthesisHandle {
   #playHandle?: PlayoutHandle;
   intFut = new Future();
   #logger = log();
-
-  constructor(speechId: string, ttsSource: SpeechSource, agentPlayout: AgentPlayout, tts: TTS) {
+  synchronizer: TextAudioSynchronizer;
+
+  constructor(
+    speechId: string,
+    ttsSource: SpeechSource,
+    agentPlayout: AgentPlayout,
+    tts: TTS,
+    synchronizer: TextAudioSynchronizer,
+  ) {
     this.#speechId = speechId;
     this.ttsSource = ttsSource;
     this.#agentPlayout = agentPlayout;
     this.tts = tts;
+    this.synchronizer = synchronizer;
   }
 
   get speechId(): string {
@@ -51,7 +60,7 @@ export class SynthesisHandle {
       throw new Error('synthesis was interrupted');
     }
 
-    this.#playHandle = this.#agentPlayout.play(this.#speechId, this.queue);
+    this.#playHandle = this.#agentPlayout.play(this.#speechId, this.queue, this.synchronizer);
     return this.#playHandle;
   }
 
@@ -86,8 +95,18 @@ export class AgentOutput {
     await Promise.all(this.#tasks);
   }
 
-  synthesize(speechId: string, ttsSource: SpeechSource): SynthesisHandle {
-    const handle = new SynthesisHandle(speechId, ttsSource, this.#agentPlayout, this.#tts);
+  synthesize(
+    speechId: string,
+    ttsSource: SpeechSource,
+    synchronizer: TextAudioSynchronizer,
+  ): SynthesisHandle {
+    const handle = new SynthesisHandle(
+      speechId,
+      ttsSource,
+      this.#agentPlayout,
+      this.#tts,
+      synchronizer,
+    );
     const task = this.#synthesize(handle);
     this.#tasks.push(task);
     task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
@@ -136,6 +155,8 @@ const stringSynthesisTask = (text: string, handle: SynthesisHandle): Cancellable
 
     const ttsStream = handle.tts.stream();
     ttsStream.pushText(text);
+    handle.synchronizer.pushText(text);
+    handle.synchronizer.markTextSegmentEnd();
     ttsStream.flush();
     ttsStream.endInput();
     for await (const audio of ttsStream) {
@@ -178,8 +199,16 @@ const streamSynthesisTask = (
     for await (const text of stream) {
       fullText += text;
       if (cancelled) break;
+      handle.synchronizer.pushText(text);
       ttsStream.pushText(text);
     }
+    handle.synchronizer.markTextSegmentEnd();
+
+    // end the audio queue early if there is no actual text to turn into speech
+    if (!fullText || fullText.trim().length === 0) {
+      cancelled = true;
+      handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
+    }
     ttsStream.flush();
     ttsStream.endInput();
 

package/src/pipeline/agent_playout.ts

@@ -5,6 +5,7 @@ import type { AudioFrame, AudioSource } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import EventEmitter from 'node:events';
 import { log } from '../log.js';
+import type { TextAudioSynchronizer } from '../transcription.js';
 import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';
 import { SynthesisHandle } from './agent_output.js';
 
@@ -23,6 +24,7 @@ export class PlayoutHandle {
   #audioSource: AudioSource;
   playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
   totalPlayedTime?: number;
+  synchronizer: TextAudioSynchronizer;
   #interrupted = false;
   pushedDuration = 0;
   intFut = new Future();
@@ -32,10 +34,12 @@ export class PlayoutHandle {
     speechId: string,
     audioSource: AudioSource,
     playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+    synchronizer: TextAudioSynchronizer,
   ) {
     this.#speechId = speechId;
     this.#audioSource = audioSource;
     this.playoutSource = playoutSource;
+    this.synchronizer = synchronizer;
   }
 
   get speechId(): string {
@@ -91,12 +95,13 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
   play(
     speechId: string,
     playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+    synchronizer: TextAudioSynchronizer,
   ): PlayoutHandle {
     if (this.#closed) {
       throw new Error('source closed');
     }
 
-    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource, synchronizer);
 
     this.#playoutTask = this.#playout(handle, this.#playoutTask);
     return handle;
@@ -109,6 +114,7 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
         handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
 
         if (handle.interrupted || captureTask.error) {
+          handle.synchronizer.close(true);
           this.#audioSource.clearQueue(); // make sure to remove any queued frames
         }
 
@@ -156,9 +162,11 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
               .child({ speechId: handle.speechId })
               .debug('started playing the first time');
             this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);
+            handle.synchronizer.segmentPlayoutStarted();
             firstFrame = false;
           }
           handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;
+          handle.synchronizer.pushAudio(frame);
           await this.#audioSource.captureFrame(frame);
           await this.#audioSource.waitForPlayout();
         }
@@ -170,6 +178,7 @@ export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentP
         //   await this.#audioSource.waitForPlayout();
         // }
 
+        handle.synchronizer.close(false);
         resolve();
       });
 

package/src/pipeline/human_input.ts

@@ -57,6 +57,14 @@ export class HumanInput extends (EventEmitter as new () => TypedEmitter<HumanInp
     this.#subscribeToMicrophone();
   }
 
+  get participant(): RemoteParticipant {
+    return this.#participant;
+  }
+
+  get subscribedTrack(): RemoteAudioTrack | undefined {
+    return this.#subscribedTrack;
+  }
+
   #subscribeToMicrophone(): void {
     if (!this.#participant) {
       this.#logger.error('Participant is not set');