@livekit/agents 0.2.0 → 0.3.0

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@livekit/agents",
-  "version": "0.2.0",
-  "description": "LiveKit Node Agents",
+  "version": "0.3.0",
+  "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
-  "author": "aoife cassidy <aoife@livekit.io>",
+  "author": "LiveKit",
   "type": "module",
   "devDependencies": {
     "@microsoft/api-extractor": "^7.35.0",
@@ -14,7 +14,7 @@
   },
   "dependencies": {
     "@livekit/protocol": "^1.21.0",
-    "@livekit/rtc-node": "^0.8.1",
+    "@livekit/rtc-node": "^0.9.0",
     "commander": "^12.0.0",
     "livekit-server-sdk": "^2.6.1",
     "pino": "^8.19.0",
@@ -24,6 +24,8 @@
   },
   "scripts": {
     "build": "tsc",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
     "lint": "eslint -f unix \"src/**/*.ts\"",
     "api:check": "api-extractor run --typescript-compiler-folder ../node_modules/typescript",
     "api:update": "api-extractor run --local --typescript-compiler-folder ../node_modules/typescript --verbose"

package/src/audio.ts

@@ -6,36 +6,36 @@ import { log } from './log.js';
 
 /** AudioByteStream translates between LiveKit AudioFrame packets and raw byte data. */
 export class AudioByteStream {
-  private sampleRate: number;
-  private numChannels: number;
-  private bytesPerFrame: number;
-  private buf: Int8Array;
+  #sampleRate: number;
+  #numChannels: number;
+  #bytesPerFrame: number;
+  #buf: Int8Array;
 
   constructor(sampleRate: number, numChannels: number, samplesPerChannel: number | null = null) {
-    this.sampleRate = sampleRate;
-    this.numChannels = numChannels;
+    this.#sampleRate = sampleRate;
+    this.#numChannels = numChannels;
 
     if (samplesPerChannel === null) {
       samplesPerChannel = Math.floor(sampleRate / 50); // 20ms by default
     }
 
-    this.bytesPerFrame = numChannels * samplesPerChannel * 2; // 2 bytes per sample (Int16)
-    this.buf = new Int8Array();
+    this.#bytesPerFrame = numChannels * samplesPerChannel * 2; // 2 bytes per sample (Int16)
+    this.#buf = new Int8Array();
   }
 
   write(data: ArrayBuffer): AudioFrame[] {
-    this.buf = new Int8Array([...this.buf, ...new Int8Array(data)]);
+    this.#buf = new Int8Array([...this.#buf, ...new Int8Array(data)]);
 
     const frames: AudioFrame[] = [];
-    while (this.buf.length >= this.bytesPerFrame) {
-      const frameData = this.buf.slice(0, this.bytesPerFrame);
-      this.buf = this.buf.slice(this.bytesPerFrame);
+    while (this.#buf.length >= this.#bytesPerFrame) {
+      const frameData = this.#buf.slice(0, this.#bytesPerFrame);
+      this.#buf = this.#buf.slice(this.#bytesPerFrame);
 
       frames.push(
         new AudioFrame(
           new Int16Array(frameData.buffer),
-          this.sampleRate,
-          this.numChannels,
+          this.#sampleRate,
+          this.#numChannels,
           frameData.length / 2,
         ),
       );
@@ -45,17 +45,17 @@ export class AudioByteStream {
   }
 
   flush(): AudioFrame[] {
-    if (this.buf.length % (2 * this.numChannels) !== 0) {
+    if (this.#buf.length % (2 * this.#numChannels) !== 0) {
       log().warn('AudioByteStream: incomplete frame during flush, dropping');
       return [];
     }
 
     return [
       new AudioFrame(
-        new Int16Array(this.buf.buffer),
-        this.sampleRate,
-        this.numChannels,
-        this.buf.length / 2,
+        new Int16Array(this.#buf.buffer),
+        this.#sampleRate,
+        this.#numChannels,
+        this.#buf.length / 2,
       ),
     ];
   }

package/src/cli.ts

@@ -5,7 +5,7 @@ import { Command, Option } from 'commander';
 import type { EventEmitter } from 'events';
 import { initializeLogger, log } from './log.js';
 import { version } from './version.js';
-import { Worker, type WorkerOptions } from './worker.js';
+import { Worker, WorkerOptions } from './worker.js';
 
 type CliArgs = {
   opts: WorkerOptions;
@@ -18,7 +18,10 @@ type CliArgs = {
 
 const runWorker = async (args: CliArgs) => {
   initializeLogger({ pretty: !args.production, level: args.opts.logLevel });
-  const worker = new Worker(args.opts);
+
+  // though `production` is defined in WorkerOptions, it will always be overriddden by CLI.
+  const { production: _, ...opts } = args.opts; // eslint-disable-line @typescript-eslint/no-unused-vars
+  const worker = new Worker(new WorkerOptions({ production: args.production, ...opts }));
 
   if (args.room) {
     worker.event.once('worker_registered', () => {
@@ -72,20 +75,29 @@ export const runApp = (opts: WorkerOptions) => {
         .env('LOG_LEVEL'),
     )
     .addOption(
-      new Option('--url <string>', 'LiveKit server or Cloud project websocket URL')
-        .makeOptionMandatory(true)
-        .env('LIVEKIT_URL'),
+      new Option('--url <string>', 'LiveKit server or Cloud project websocket URL').env(
+        'LIVEKIT_URL',
+      ),
     )
     .addOption(
-      new Option('--api-key <string>', "LiveKit server or Cloud project's API key")
-        .makeOptionMandatory(true)
-        .env('LIVEKIT_API_KEY'),
+      new Option('--api-key <string>', "LiveKit server or Cloud project's API key").env(
+        'LIVEKIT_API_KEY',
+      ),
     )
     .addOption(
-      new Option('--api-secret <string>', "LiveKit server or Cloud project's API secret")
-        .makeOptionMandatory(true)
-        .env('LIVEKIT_API_SECRET'),
-    );
+      new Option('--api-secret <string>', "LiveKit server or Cloud project's API secret").env(
+        'LIVEKIT_API_SECRET',
+      ),
+    )
+    .action(() => {
+      if (
+        // do not run CLI if origin file is agents/ipc/job_main.js
+        process.argv[1] !== new URL('ipc/job_main.js', import.meta.url).pathname ||
+        process.argv.length < 3
+      ) {
+        program.help();
+      }
+    });
 
   program
     .command('start')
@@ -106,6 +118,12 @@ export const runApp = (opts: WorkerOptions) => {
   program
     .command('dev')
     .description('Start the worker in development mode')
+    .addOption(
+      new Option('--log-level <level>', 'Set the logging level')
+        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
+        .default('debug')
+        .env('LOG_LEVEL'),
+    )
     .action(() => {
       const options = program.optsWithGlobals();
       opts.wsURL = options.url || opts.wsURL;
@@ -123,7 +141,13 @@ export const runApp = (opts: WorkerOptions) => {
     .command('connect')
     .description('Connect to a specific room')
     .requiredOption('--room <string>', 'Room name to connect to')
-    .option('--participant-identity <string>', 'Participant identitiy to connect as')
+    .option('--participant-identity <string>', 'Identity of user to listen to')
+    .addOption(
+      new Option('--log-level <level>', 'Set the logging level')
+        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
+        .default('debug')
+        .env('LOG_LEVEL'),
+    )
     .action((...[, command]) => {
       const options = command.optsWithGlobals();
       opts.wsURL = options.url || opts.wsURL;

package/src/generator.ts

@@ -9,6 +9,20 @@ export interface Agent {
   prewarm?: (proc: JobProcess) => unknown;
 }
 
+/** Helper to check if an object is an agent before running it.
+ *
+ * @internal
+ */
+export function isAgent(obj: unknown): obj is Agent {
+  return (
+    typeof obj === 'object' &&
+    obj !== null &&
+    'entry' in obj &&
+    typeof (obj as Agent).entry === 'function' &&
+    (('prewarm' in obj && typeof (obj as Agent).prewarm === 'function') || !('prewarm' in obj))
+  );
+}
+
 /**
  * Helper to define an agent according to the required interface.
  * @example A basic agent with entry and prewarm functions

package/src/http_server.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'http';
+import { log } from './log.js';
 
 const healthCheck = async (res: ServerResponse) => {
   res.writeHead(200);
@@ -31,6 +32,10 @@ export class HTTPServer {
     return new Promise((resolve, reject) => {
       this.app.listen(this.port, this.host, (err?: Error) => {
         if (err) reject(err);
+        const address = this.app.address();
+        if (typeof address! !== 'string') {
+          log().info(`Server is listening on port ${address!.port}`);
+        }
         resolve();
       });
     });

package/src/index.ts

@@ -11,6 +11,7 @@
  */
 import * as cli from './cli.js';
 import * as llm from './llm/index.js';
+import * as multimodal from './multimodal/index.js';
 import * as stt from './stt/index.js';
 import * as tts from './tts/index.js';
 
@@ -24,5 +25,6 @@ export * from './log.js';
 export * from './generator.js';
 export * from './tokenize.js';
 export * from './audio.js';
+export * from './transcription.js';
 
-export { cli, stt, tts, llm };
+export { cli, stt, tts, llm, multimodal };

package/src/ipc/job_main.ts

@@ -7,7 +7,7 @@ import { fork } from 'child_process';
 import { EventEmitter, once } from 'events';
 import type { Logger } from 'pino';
 import { fileURLToPath } from 'url';
-import type { Agent } from '../generator.js';
+import { type Agent, isAgent } from '../generator.js';
 import type { RunningJobInfo } from '../job.js';
 import { JobContext } from '../job.js';
 import { JobProcess } from '../job.js';
@@ -93,7 +93,14 @@ if (process.send) {
   //   [0] `node'
   //   [1] import.meta.filename
   //   [2] import.meta.filename of function containing entry file
-  const agent: Agent = await import(process.argv[2]).then((agent) => agent.default);
+  const moduleFile = process.argv[2];
+  const agent: Agent = await import(moduleFile).then((module) => {
+    const agent = module.default;
+    if (agent === undefined || !isAgent(agent)) {
+      throw new Error(`Unable to load agent: Missing or invalid default export in ${moduleFile}`);
+    }
+    return agent;
+  });
   if (!agent.prewarm) {
     agent.prewarm = defaultInitializeProcessFunc;
   }

package/src/ipc/proc_pool.ts

@@ -102,6 +102,7 @@ export class ProcPool {
     }
     this.closed = true;
     this.controller.abort();
+    this.warmedProcQueue.items.forEach((e) => e.close());
     this.executors.forEach((e) => e.close());
     await Promise.allSettled(this.tasks);
   }

package/src/job.ts

@@ -9,7 +9,7 @@ import type {
   Room,
   RtcConfiguration,
 } from '@livekit/rtc-node';
-import { RoomEvent, TrackKind } from '@livekit/rtc-node';
+import { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';
 import type { Logger } from 'pino';
 import { log } from './log.js';
 
@@ -100,6 +100,26 @@ export class JobContext {
     this.shutdownCallbacks.push(callback);
   }
 
+  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {
+    if (!this.#room.isConnected) {
+      throw new Error('room is not connected');
+    }
+
+    console.log(this.#room.remoteParticipants.values());
+
+    for (const p of this.#room.remoteParticipants.values()) {
+      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {
+        return p;
+      }
+    }
+
+    return new Promise((resolve) => {
+      this.#room.once(RoomEvent.ParticipantConnected, () => {
+        resolve(this.#room.remoteParticipants.values().next().value);
+      });
+    });
+  }
+
   /**
    * Connects the agent to the room.
    *

package/src/multimodal/agent_playout.ts

@@ -0,0 +1,254 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { type AudioSource } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import { AudioByteStream } from '../audio.js';
+import type { TranscriptionForwarder } from '../transcription.js';
+import { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+
+export const proto = {};
+
+export class PlayoutHandle extends EventEmitter {
+  #audioSource: AudioSource;
+  #sampleRate: number;
+  #itemId: string;
+  #contentIndex: number;
+  /** @internal */
+  transcriptionFwd: TranscriptionForwarder;
+  /** @internal */
+  doneFut: Future;
+  /** @internal */
+  intFut: Future;
+  /** @internal */
+  #interrupted: boolean;
+  /** @internal */
+  pushedDuration: number;
+  /** @internal */
+  totalPlayedTime: number | undefined; // Set when playout is done
+
+  constructor(
+    audioSource: AudioSource,
+    sampleRate: number,
+    itemId: string,
+    contentIndex: number,
+    transcriptionFwd: TranscriptionForwarder,
+  ) {
+    super();
+    this.#audioSource = audioSource;
+    this.#sampleRate = sampleRate;
+    this.#itemId = itemId;
+    this.#contentIndex = contentIndex;
+    this.transcriptionFwd = transcriptionFwd;
+    this.doneFut = new Future();
+    this.intFut = new Future();
+    this.#interrupted = false;
+    this.pushedDuration = 0;
+    this.totalPlayedTime = undefined;
+  }
+
+  get itemId(): string {
+    return this.#itemId;
+  }
+
+  get audioSamples(): number {
+    if (this.totalPlayedTime !== undefined) {
+      return Math.floor(this.totalPlayedTime * this.#sampleRate);
+    }
+
+    return Math.floor(this.pushedDuration - this.#audioSource.queuedDuration * this.#sampleRate);
+  }
+
+  get textChars(): number {
+    return this.transcriptionFwd.currentCharacterIndex;
+  }
+
+  get contentIndex(): number {
+    return this.#contentIndex;
+  }
+
+  get interrupted(): boolean {
+    return this.#interrupted;
+  }
+
+  get done(): boolean {
+    return this.doneFut.done || this.#interrupted;
+  }
+
+  interrupt() {
+    if (this.doneFut.done) return;
+    this.intFut.resolve();
+    this.#interrupted = true;
+  }
+}
+
+export class AgentPlayout {
+  #audioSource: AudioSource;
+  #playoutTask: CancellablePromise<void> | null;
+  #sampleRate: number;
+  #numChannels: number;
+  #inFrameSize: number;
+  #outFrameSize: number;
+  constructor(
+    audioSource: AudioSource,
+    sampleRate: number,
+    numChannels: number,
+    inFrameSize: number,
+    outFrameSize: number,
+  ) {
+    this.#audioSource = audioSource;
+    this.#playoutTask = null;
+    this.#sampleRate = sampleRate;
+    this.#numChannels = numChannels;
+    this.#inFrameSize = inFrameSize;
+    this.#outFrameSize = outFrameSize;
+  }
+
+  play(
+    itemId: string,
+    contentIndex: number,
+    transcriptionFwd: TranscriptionForwarder,
+    textStream: AsyncIterableQueue<string>,
+    audioStream: AsyncIterableQueue<AudioFrame>,
+  ): PlayoutHandle {
+    const handle = new PlayoutHandle(
+      this.#audioSource,
+      this.#sampleRate,
+      itemId,
+      contentIndex,
+      transcriptionFwd,
+    );
+    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
+    return handle;
+  }
+
+  #makePlayoutTask(
+    oldTask: CancellablePromise<void> | null,
+    handle: PlayoutHandle,
+    textStream: AsyncIterableQueue<string>,
+    audioStream: AsyncIterableQueue<AudioFrame>,
+  ): CancellablePromise<void> {
+    return new CancellablePromise<void>((resolve, reject, onCancel) => {
+      let cancelled = false;
+      onCancel(() => {
+        cancelled = true;
+      });
+
+      (async () => {
+        try {
+          if (oldTask) {
+            await gracefullyCancel(oldTask);
+          }
+
+          let firstFrame = true;
+
+          const readText = () =>
+            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {
+              let cancelledText = false;
+              onCancelText(() => {
+                cancelledText = true;
+              });
+
+              (async () => {
+                try {
+                  for await (const text of textStream) {
+                    if (cancelledText || cancelled) {
+                      break;
+                    }
+                    handle.transcriptionFwd.pushText(text);
+                  }
+                  resolveText();
+                } catch (error) {
+                  rejectText(error);
+                }
+              })();
+            });
+
+          const capture = () =>
+            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {
+              let cancelledCapture = false;
+              onCancelCapture(() => {
+                cancelledCapture = true;
+              });
+
+              (async () => {
+                try {
+                  const samplesPerChannel = this.#outFrameSize;
+                  const bstream = new AudioByteStream(
+                    this.#sampleRate,
+                    this.#numChannels,
+                    samplesPerChannel,
+                  );
+
+                  for await (const frame of audioStream) {
+                    if (cancelledCapture || cancelled) {
+                      break;
+                    }
+                    if (firstFrame) {
+                      handle.transcriptionFwd.start();
+                      firstFrame = false;
+                    }
+
+                    handle.transcriptionFwd.pushAudio(frame);
+
+                    for (const f of bstream.write(frame.data.buffer)) {
+                      handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                      await this.#audioSource.captureFrame(f);
+                    }
+                  }
+
+                  if (!cancelledCapture && !cancelled) {
+                    for (const f of bstream.flush()) {
+                      handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                      await this.#audioSource.captureFrame(f);
+                    }
+
+                    handle.transcriptionFwd.markAudioComplete();
+
+                    await this.#audioSource.waitForPlayout();
+                  }
+
+                  resolveCapture();
+                } catch (error) {
+                  rejectCapture(error);
+                }
+              })();
+            });
+
+          const readTextTask = readText();
+          const captureTask = capture();
+
+          try {
+            await Promise.race([captureTask, handle.intFut.await]);
+          } finally {
+            if (!captureTask.isCancelled) {
+              await gracefullyCancel(captureTask);
+            }
+
+            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+
+            if (handle.interrupted || captureTask.error) {
+              this.#audioSource.clearQueue(); // make sure to remove any queued frames
+            }
+
+            if (!readTextTask.isCancelled) {
+              await gracefullyCancel(readTextTask);
+            }
+
+            if (!firstFrame && !handle.interrupted) {
+              handle.transcriptionFwd.markTextComplete();
+            }
+
+            handle.doneFut.resolve();
+            await handle.transcriptionFwd.close(handle.interrupted);
+          }
+
+          resolve();
+        } catch (error) {
+          reject(error);
+        }
+      })();
+    });
+  }
+}

package/src/multimodal/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './multimodal_agent.js';
+export * from './agent_playout.js';