@livekit/agents 0.2.0 → 0.3.1

package/src/cli.ts

@@ -5,7 +5,7 @@ import { Command, Option } from 'commander';
 import type { EventEmitter } from 'events';
 import { initializeLogger, log } from './log.js';
 import { version } from './version.js';
-import { Worker, type WorkerOptions } from './worker.js';
+import { Worker, WorkerOptions } from './worker.js';
 
 type CliArgs = {
   opts: WorkerOptions;
@@ -18,11 +18,15 @@ type CliArgs = {
 
 const runWorker = async (args: CliArgs) => {
   initializeLogger({ pretty: !args.production, level: args.opts.logLevel });
-  const worker = new Worker(args.opts);
+  const logger = log();
+
+  // though `production` is defined in WorkerOptions, it will always be overriddden by CLI.
+  const { production: _, ...opts } = args.opts; // eslint-disable-line @typescript-eslint/no-unused-vars
+  const worker = new Worker(new WorkerOptions({ production: args.production, ...opts }));
 
   if (args.room) {
     worker.event.once('worker_registered', () => {
-      log().info(`connecting to room ${args.room}`);
+      logger.info(`connecting to room ${args.room}`);
       worker.simulateJob(args.room!, args.participantIdentity);
     });
   }
@@ -30,21 +34,21 @@ const runWorker = async (args: CliArgs) => {
   process.once('SIGINT', async () => {
     // allow C-c C-c for force interrupt
     process.once('SIGINT', () => {
-      log().info('worker closed forcefully');
+      logger.info('worker closed forcefully');
       process.exit(130); // SIGINT exit code
     });
     if (args.production) {
       await worker.drain();
     }
     await worker.close();
-    log().info('worker closed');
+    logger.info('worker closed');
     process.exit(130); // SIGINT exit code
   });
 
   try {
     await worker.run();
   } catch {
-    log().fatal('worker failed');
+    logger.fatal('worker failed');
     process.exit(1);
   }
 };
@@ -72,20 +76,29 @@ export const runApp = (opts: WorkerOptions) => {
         .env('LOG_LEVEL'),
     )
     .addOption(
-      new Option('--url <string>', 'LiveKit server or Cloud project websocket URL')
-        .makeOptionMandatory(true)
-        .env('LIVEKIT_URL'),
+      new Option('--url <string>', 'LiveKit server or Cloud project websocket URL').env(
+        'LIVEKIT_URL',
+      ),
     )
     .addOption(
-      new Option('--api-key <string>', "LiveKit server or Cloud project's API key")
-        .makeOptionMandatory(true)
-        .env('LIVEKIT_API_KEY'),
+      new Option('--api-key <string>', "LiveKit server or Cloud project's API key").env(
+        'LIVEKIT_API_KEY',
+      ),
     )
     .addOption(
-      new Option('--api-secret <string>', "LiveKit server or Cloud project's API secret")
-        .makeOptionMandatory(true)
-        .env('LIVEKIT_API_SECRET'),
-    );
+      new Option('--api-secret <string>', "LiveKit server or Cloud project's API secret").env(
+        'LIVEKIT_API_SECRET',
+      ),
+    )
+    .action(() => {
+      if (
+        // do not run CLI if origin file is agents/ipc/job_main.js
+        process.argv[1] !== new URL('ipc/job_main.js', import.meta.url).pathname ||
+        process.argv.length < 3
+      ) {
+        program.help();
+      }
+    });
 
   program
     .command('start')
@@ -106,6 +119,12 @@ export const runApp = (opts: WorkerOptions) => {
   program
     .command('dev')
     .description('Start the worker in development mode')
+    .addOption(
+      new Option('--log-level <level>', 'Set the logging level')
+        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
+        .default('debug')
+        .env('LOG_LEVEL'),
+    )
     .action(() => {
       const options = program.optsWithGlobals();
       opts.wsURL = options.url || opts.wsURL;
@@ -123,7 +142,13 @@ export const runApp = (opts: WorkerOptions) => {
     .command('connect')
     .description('Connect to a specific room')
     .requiredOption('--room <string>', 'Room name to connect to')
-    .option('--participant-identity <string>', 'Participant identitiy to connect as')
+    .option('--participant-identity <string>', 'Identity of user to listen to')
+    .addOption(
+      new Option('--log-level <level>', 'Set the logging level')
+        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
+        .default('debug')
+        .env('LOG_LEVEL'),
+    )
     .action((...[, command]) => {
       const options = command.optsWithGlobals();
       opts.wsURL = options.url || opts.wsURL;

package/src/generator.ts

@@ -9,6 +9,20 @@ export interface Agent {
   prewarm?: (proc: JobProcess) => unknown;
 }
 
+/** Helper to check if an object is an agent before running it.
+ *
+ * @internal
+ */
+export function isAgent(obj: unknown): obj is Agent {
+  return (
+    typeof obj === 'object' &&
+    obj !== null &&
+    'entry' in obj &&
+    typeof (obj as Agent).entry === 'function' &&
+    (('prewarm' in obj && typeof (obj as Agent).prewarm === 'function') || !('prewarm' in obj))
+  );
+}
+
 /**
  * Helper to define an agent according to the required interface.
  * @example A basic agent with entry and prewarm functions

package/src/http_server.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'http';
+import { log } from './log.js';
 
 const healthCheck = async (res: ServerResponse) => {
   res.writeHead(200);
@@ -12,6 +13,7 @@ export class HTTPServer {
   host: string;
   port: number;
   app: Server;
+  #logger = log();
 
   constructor(host: string, port: number) {
     this.host = host;
@@ -31,6 +33,10 @@ export class HTTPServer {
     return new Promise((resolve, reject) => {
       this.app.listen(this.port, this.host, (err?: Error) => {
         if (err) reject(err);
+        const address = this.app.address();
+        if (typeof address! !== 'string') {
+          this.#logger.info(`Server is listening on port ${address!.port}`);
+        }
         resolve();
       });
     });

package/src/index.ts

@@ -11,6 +11,7 @@
  */
 import * as cli from './cli.js';
 import * as llm from './llm/index.js';
+import * as multimodal from './multimodal/index.js';
 import * as stt from './stt/index.js';
 import * as tts from './tts/index.js';
 
@@ -24,5 +25,6 @@ export * from './log.js';
 export * from './generator.js';
 export * from './tokenize.js';
 export * from './audio.js';
+export * from './transcription.js';
 
-export { cli, stt, tts, llm };
+export { cli, stt, tts, llm, multimodal };

package/src/ipc/job_main.ts

@@ -7,7 +7,7 @@ import { fork } from 'child_process';
 import { EventEmitter, once } from 'events';
 import type { Logger } from 'pino';
 import { fileURLToPath } from 'url';
-import type { Agent } from '../generator.js';
+import { type Agent, isAgent } from '../generator.js';
 import type { RunningJobInfo } from '../job.js';
 import { JobContext } from '../job.js';
 import { JobProcess } from '../job.js';
@@ -93,7 +93,14 @@ if (process.send) {
   //   [0] `node'
   //   [1] import.meta.filename
   //   [2] import.meta.filename of function containing entry file
-  const agent: Agent = await import(process.argv[2]).then((agent) => agent.default);
+  const moduleFile = process.argv[2];
+  const agent: Agent = await import(moduleFile).then((module) => {
+    const agent = module.default;
+    if (agent === undefined || !isAgent(agent)) {
+      throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);
+    }
+    return agent;
+  });
   if (!agent.prewarm) {
     agent.prewarm = defaultInitializeProcessFunc;
   }

package/src/ipc/proc_pool.ts

@@ -102,6 +102,7 @@ export class ProcPool {
     }
     this.closed = true;
     this.controller.abort();
+    this.warmedProcQueue.items.forEach((e) => e.close());
     this.executors.forEach((e) => e.close());
     await Promise.allSettled(this.tasks);
   }

package/src/job.ts

@@ -9,7 +9,7 @@ import type {
   Room,
   RtcConfiguration,
 } from '@livekit/rtc-node';
-import { RoomEvent, TrackKind } from '@livekit/rtc-node';
+import { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';
 import type { Logger } from 'pino';
 import { log } from './log.js';
 
@@ -100,6 +100,42 @@ export class JobContext {
     this.shutdownCallbacks.push(callback);
   }
 
+  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {
+    if (!this.#room.isConnected) {
+      throw new Error('room is not connected');
+    }
+
+    for (const p of this.#room.remoteParticipants.values()) {
+      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {
+        return p;
+      }
+    }
+
+    return new Promise((resolve, reject) => {
+      const onParticipantConnected = (participant: RemoteParticipant) => {
+        if (
+          (!identity || participant.identity === identity) &&
+          participant.info.kind != ParticipantKind.AGENT
+        ) {
+          clearHandlers();
+          resolve(participant);
+        }
+      };
+      const onDisconnected = () => {
+        clearHandlers();
+        reject(new Error('Room disconnected while waiting for participant'));
+      };
+
+      const clearHandlers = () => {
+        this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);
+        this.#room.off(RoomEvent.Disconnected, onDisconnected);
+      };
+
+      this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);
+      this.#room.on(RoomEvent.Disconnected, onDisconnected);
+    });
+  }
+
   /**
    * Connects the agent to the room.
    *

package/src/multimodal/agent_playout.ts

@@ -0,0 +1,254 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { type AudioSource } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import { AudioByteStream } from '../audio.js';
+import type { TranscriptionForwarder } from '../transcription.js';
+import { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+
+export const proto = {};
+
+export class PlayoutHandle extends EventEmitter {
+  #audioSource: AudioSource;
+  #sampleRate: number;
+  #itemId: string;
+  #contentIndex: number;
+  /** @internal */
+  transcriptionFwd: TranscriptionForwarder;
+  /** @internal */
+  doneFut: Future;
+  /** @internal */
+  intFut: Future;
+  /** @internal */
+  #interrupted: boolean;
+  /** @internal */
+  pushedDuration: number;
+  /** @internal */
+  totalPlayedTime: number | undefined; // Set when playout is done
+
+  constructor(
+    audioSource: AudioSource,
+    sampleRate: number,
+    itemId: string,
+    contentIndex: number,
+    transcriptionFwd: TranscriptionForwarder,
+  ) {
+    super();
+    this.#audioSource = audioSource;
+    this.#sampleRate = sampleRate;
+    this.#itemId = itemId;
+    this.#contentIndex = contentIndex;
+    this.transcriptionFwd = transcriptionFwd;
+    this.doneFut = new Future();
+    this.intFut = new Future();
+    this.#interrupted = false;
+    this.pushedDuration = 0;
+    this.totalPlayedTime = undefined;
+  }
+
+  get itemId(): string {
+    return this.#itemId;
+  }
+
+  get audioSamples(): number {
+    if (this.totalPlayedTime !== undefined) {
+      return Math.floor(this.totalPlayedTime * this.#sampleRate);
+    }
+
+    return Math.floor(this.pushedDuration - this.#audioSource.queuedDuration * this.#sampleRate);
+  }
+
+  get textChars(): number {
+    return this.transcriptionFwd.currentCharacterIndex;
+  }
+
+  get contentIndex(): number {
+    return this.#contentIndex;
+  }
+
+  get interrupted(): boolean {
+    return this.#interrupted;
+  }
+
+  get done(): boolean {
+    return this.doneFut.done || this.#interrupted;
+  }
+
+  interrupt() {
+    if (this.doneFut.done) return;
+    this.intFut.resolve();
+    this.#interrupted = true;
+  }
+}
+
+export class AgentPlayout {
+  #audioSource: AudioSource;
+  #playoutTask: CancellablePromise<void> | null;
+  #sampleRate: number;
+  #numChannels: number;
+  #inFrameSize: number;
+  #outFrameSize: number;
+  constructor(
+    audioSource: AudioSource,
+    sampleRate: number,
+    numChannels: number,
+    inFrameSize: number,
+    outFrameSize: number,
+  ) {
+    this.#audioSource = audioSource;
+    this.#playoutTask = null;
+    this.#sampleRate = sampleRate;
+    this.#numChannels = numChannels;
+    this.#inFrameSize = inFrameSize;
+    this.#outFrameSize = outFrameSize;
+  }
+
+  play(
+    itemId: string,
+    contentIndex: number,
+    transcriptionFwd: TranscriptionForwarder,
+    textStream: AsyncIterableQueue<string>,
+    audioStream: AsyncIterableQueue<AudioFrame>,
+  ): PlayoutHandle {
+    const handle = new PlayoutHandle(
+      this.#audioSource,
+      this.#sampleRate,
+      itemId,
+      contentIndex,
+      transcriptionFwd,
+    );
+    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
+    return handle;
+  }
+
+  #makePlayoutTask(
+    oldTask: CancellablePromise<void> | null,
+    handle: PlayoutHandle,
+    textStream: AsyncIterableQueue<string>,
+    audioStream: AsyncIterableQueue<AudioFrame>,
+  ): CancellablePromise<void> {
+    return new CancellablePromise<void>((resolve, reject, onCancel) => {
+      let cancelled = false;
+      onCancel(() => {
+        cancelled = true;
+      });
+
+      (async () => {
+        try {
+          if (oldTask) {
+            await gracefullyCancel(oldTask);
+          }
+
+          let firstFrame = true;
+
+          const readText = () =>
+            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {
+              let cancelledText = false;
+              onCancelText(() => {
+                cancelledText = true;
+              });
+
+              (async () => {
+                try {
+                  for await (const text of textStream) {
+                    if (cancelledText || cancelled) {
+                      break;
+                    }
+                    handle.transcriptionFwd.pushText(text);
+                  }
+                  resolveText();
+                } catch (error) {
+                  rejectText(error);
+                }
+              })();
+            });
+
+          const capture = () =>
+            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {
+              let cancelledCapture = false;
+              onCancelCapture(() => {
+                cancelledCapture = true;
+              });
+
+              (async () => {
+                try {
+                  const samplesPerChannel = this.#outFrameSize;
+                  const bstream = new AudioByteStream(
+                    this.#sampleRate,
+                    this.#numChannels,
+                    samplesPerChannel,
+                  );
+
+                  for await (const frame of audioStream) {
+                    if (cancelledCapture || cancelled) {
+                      break;
+                    }
+                    if (firstFrame) {
+                      handle.transcriptionFwd.start();
+                      firstFrame = false;
+                    }
+
+                    handle.transcriptionFwd.pushAudio(frame);
+
+                    for (const f of bstream.write(frame.data.buffer)) {
+                      handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                      await this.#audioSource.captureFrame(f);
+                    }
+                  }
+
+                  if (!cancelledCapture && !cancelled) {
+                    for (const f of bstream.flush()) {
+                      handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                      await this.#audioSource.captureFrame(f);
+                    }
+
+                    handle.transcriptionFwd.markAudioComplete();
+
+                    await this.#audioSource.waitForPlayout();
+                  }
+
+                  resolveCapture();
+                } catch (error) {
+                  rejectCapture(error);
+                }
+              })();
+            });
+
+          const readTextTask = readText();
+          const captureTask = capture();
+
+          try {
+            await Promise.race([captureTask, handle.intFut.await]);
+          } finally {
+            if (!captureTask.isCancelled) {
+              await gracefullyCancel(captureTask);
+            }
+
+            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+
+            if (handle.interrupted || captureTask.error) {
+              this.#audioSource.clearQueue(); // make sure to remove any queued frames
+            }
+
+            if (!readTextTask.isCancelled) {
+              await gracefullyCancel(readTextTask);
+            }
+
+            if (!firstFrame && !handle.interrupted) {
+              handle.transcriptionFwd.markTextComplete();
+            }
+
+            handle.doneFut.resolve();
+            await handle.transcriptionFwd.close(handle.interrupted);
+          }
+
+          resolve();
+        } catch (error) {
+          reject(error);
+        }
+      })();
+    });
+  }
+}

package/src/multimodal/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './multimodal_agent.js';
+export * from './agent_playout.js';