@livekit/agents 0.1.0 → 0.3.0

package/src/job.ts

@@ -0,0 +1,278 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type * as proto from '@livekit/protocol';
+import type {
+  E2EEOptions,
+  LocalParticipant,
+  RemoteParticipant,
+  Room,
+  RtcConfiguration,
+} from '@livekit/rtc-node';
+import { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';
+import type { Logger } from 'pino';
+import { log } from './log.js';
+
+/** Which tracks, if any, should the agent automatically subscribe to? */
+export enum AutoSubscribe {
+  SUBSCRIBE_ALL,
+  SUBSCRIBE_NONE,
+  VIDEO_ONLY,
+  AUDIO_ONLY,
+}
+
+export type JobAcceptArguments = {
+  name: string;
+  identity: string;
+  metadata: string;
+};
+
+export type RunningJobInfo = {
+  acceptArguments: JobAcceptArguments;
+  job: proto.Job;
+  url: string;
+  token: string;
+};
+
+/** Attempted to add a function callback, but the function already exists. */
+export class FunctionExistsError extends Error {
+  constructor(msg?: string) {
+    super(msg);
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+}
+
+/** The job and environment context as seen by the agent, accessible by the entrypoint function. */
+export class JobContext {
+  #proc: JobProcess;
+  #info: RunningJobInfo;
+  #room: Room;
+  #onConnect: () => void;
+  #onShutdown: (s: string) => void;
+  /** @internal */
+  shutdownCallbacks: (() => Promise<void>)[] = [];
+  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];
+  #participantTasks: {
+    [id: string]: {
+      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;
+      result: Promise<void>;
+    };
+  } = {};
+  #logger: Logger;
+
+  constructor(
+    proc: JobProcess,
+    info: RunningJobInfo,
+    room: Room,
+    onConnect: () => void,
+    onShutdown: (s: string) => void,
+  ) {
+    this.#proc = proc;
+    this.#info = info;
+    this.#room = room;
+    this.#onConnect = onConnect;
+    this.#onShutdown = onShutdown;
+    this.onParticipantConnected = this.onParticipantConnected.bind(this);
+    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
+    this.#logger = log().child({ info: this.#info });
+  }
+
+  get proc(): JobProcess {
+    return this.#proc;
+  }
+
+  get job(): proto.Job {
+    return this.#info.job;
+  }
+
+  /** @returns The room the agent was called into */
+  get room(): Room {
+    return this.#room;
+  }
+
+  /** @returns The agent's participant if connected to the room, otherwise `undefined` */
+  get agent(): LocalParticipant | undefined {
+    return this.#room.localParticipant;
+  }
+
+  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */
+  addShutdownCallback(callback: () => Promise<void>) {
+    this.shutdownCallbacks.push(callback);
+  }
+
+  async waitForParticipant(identity?: string): Promise<RemoteParticipant> {
+    if (!this.#room.isConnected) {
+      throw new Error('room is not connected');
+    }
+
+    console.log(this.#room.remoteParticipants.values());
+
+    for (const p of this.#room.remoteParticipants.values()) {
+      if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {
+        return p;
+      }
+    }
+
+    return new Promise((resolve) => {
+      this.#room.once(RoomEvent.ParticipantConnected, () => {
+        resolve(this.#room.remoteParticipants.values().next().value);
+      });
+    });
+  }
+
+  /**
+   * Connects the agent to the room.
+   *
+   * @remarks
+   * It is recommended to run this command as early in the function as possible, as executing it
+   * later may cause noticeable delay between user and agent joins.
+   *
+   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |
+   * @livekit/rtc-node} for more information about the parameters.
+   */
+  async connect(
+    e2ee?: E2EEOptions,
+    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,
+    rtcConfig?: RtcConfiguration,
+  ) {
+    const opts = {
+      e2ee,
+      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,
+      rtcConfig,
+      dynacast: false,
+    };
+
+    await this.#room.connect(this.#info.url, this.#info.token, opts);
+    this.#onConnect();
+
+    this.#room.remoteParticipants.forEach(this.onParticipantConnected);
+
+    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {
+      this.#room.remoteParticipants.forEach((p) => {
+        p.trackPublications.forEach((pub) => {
+          if (
+            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||
+            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)
+          ) {
+            pub.setSubscribed(true);
+          }
+        });
+      });
+    }
+  }
+
+  /**
+   * Gracefully shuts down the job, and runs all shutdown promises.
+   *
+   * @param reason - Optional reason for shutdown
+   */
+  shutdown(reason = '') {
+    this.#onShutdown(reason);
+  }
+
+  /** @internal */
+  onParticipantConnected(p: RemoteParticipant) {
+    for (const callback of this.#participantEntrypoints) {
+      if (
+        p.identity in this.#participantTasks &&
+        this.#participantTasks[p.identity].callback == callback
+      ) {
+        this.#logger.warn(
+          'a participant has joined before a prior prticipant task matching the same identity has finished:',
+          p.identity,
+        );
+      }
+      const result = callback(this, p);
+      result.finally(() => delete this.#participantTasks[p.identity]);
+      this.#participantTasks[p.identity] = { callback, result };
+    }
+  }
+
+  /**
+   * Adds a promise to be awaited whenever a new participant joins the room.
+   *
+   * @throws {@link FunctionExistsError} if an entrypoint already exists
+   */
+  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {
+    if (this.#participantEntrypoints.includes(callback)) {
+      throw new FunctionExistsError('entrypoints cannot be added more than once');
+    }
+
+    this.#participantEntrypoints.push(callback);
+  }
+}
+
+export class JobProcess {
+  #pid = process.pid;
+  #userData: { [id: string]: unknown } = {};
+
+  get pid(): number {
+    return this.#pid;
+  }
+
+  get userData(): { [id: string]: unknown } {
+    return this.#userData;
+  }
+}
+
+/**
+ * A request sent by the server to spawn a new agent job.
+ *
+ * @remarks
+ * For most applications, this is best left to the default, which simply accepts the job and
+ * handles the logic inside the entrypoint function. This class is useful for vetting which
+ * requests should fill idle processes and which should be outright rejected.
+ */
+export class JobRequest {
+  #job: proto.Job;
+  #onReject: () => Promise<void>;
+  #onAccept: (args: JobAcceptArguments) => Promise<void>;
+
+  /** @internal */
+  constructor(
+    job: proto.Job,
+    onReject: () => Promise<void>,
+    onAccept: (args: JobAcceptArguments) => Promise<void>,
+  ) {
+    this.#job = job;
+    this.#onReject = onReject;
+    this.#onAccept = onAccept;
+  }
+
+  /** @returns The ID of the job, set by the LiveKit server */
+  get id(): string {
+    return this.#job.id;
+  }
+
+  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+  get job(): proto.Job {
+    return this.#job;
+  }
+
+  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+  get room(): proto.Room | undefined {
+    return this.#job.room;
+  }
+
+  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+  get publisher(): proto.ParticipantInfo | undefined {
+    return this.#job.participant;
+  }
+
+  /** @returns The agent's name, as set in {@link WorkerOptions} */
+  get agentName(): string {
+    return this.#job.agentName;
+  }
+
+  /** Rejects the job. */
+  async reject() {
+    await this.#onReject();
+  }
+
+  /** Accepts the job, launching it on an idle child process. */
+  async accept(name = '', identity = '', metadata = '') {
+    if (identity === '') identity = 'agent-' + this.id;
+
+    this.#onAccept({ name, identity, metadata });
+  }
+}

package/src/llm/function_context.ts

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { z } from 'zod';
+
+// heavily inspired by Vercel AI's `tool()`:
+// https://github.com/vercel/ai/blob/3b0983b/packages/ai/core/tool/tool.ts
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+/** Type reinforcement for the callable function's execute parameters. */
+export type inferParameters<P extends z.ZodTypeAny> = z.infer<P>;
+
+/** A definition for a function callable by the LLM. */
+export interface CallableFunction<P extends z.ZodTypeAny = any, R = any> {
+  description: string;
+  parameters: P;
+  execute: (args: inferParameters<P>) => PromiseLike<R>;
+}
+
+/** An object containing callable functions and their names */
+export type FunctionContext = {
+  [name: string]: CallableFunction;
+};
+
+/** @internal */
+export const oaiParams = (p: z.AnyZodObject) => {
+  const properties: Record<string, any> = {};
+  const required_properties: string[] = [];
+
+  for (const key in p.shape) {
+    const field = p.shape[key];
+    const description = field._def.description || undefined;
+    let type: string;
+    let enumValues: any[] | undefined;
+
+    if (field instanceof z.ZodEnum) {
+      enumValues = field._def.values;
+      type = typeof enumValues![0];
+    } else {
+      type = field._def.typeName.toLowerCase();
+    }
+
+    properties[key] = {
+      type: type.includes('zod') ? type.substring(3) : type,
+      description,
+      enum: enumValues,
+    };
+
+    if (!field._def.defaultValue) {
+      required_properties.push(key);
+    }
+  }
+
+  const type = 'object' as const;
+  return {
+    type,
+    properties,
+    required_properties,
+  };
+};

package/src/llm/index.ts

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type CallableFunction,
+  type FunctionContext,
+  type inferParameters,
+  oaiParams,
+} from './function_context.js';
+
+export { CallableFunction, FunctionContext, inferParameters, oaiParams };

package/src/log.ts

@@ -1,13 +1,45 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { Logger } from 'pino';
 import { pino } from 'pino';
 
-export const log = pino({
-  transport: {
-    target: 'pino-pretty',
-    options: {
-      colorize: true,
-    },
-  },
-});
+/** @internal */
+export type LoggerOptions = {
+  pretty: boolean;
+  level?: string;
+};
+
+/** @internal */
+export let loggerOptions: LoggerOptions;
+
+/** @internal */
+let logger: Logger | undefined = undefined;
+
+/** @internal */
+export const log = () => {
+  if (!logger) {
+    throw new TypeError('logger not initialized. did you forget to run initializeLogger()?');
+  }
+  return logger;
+};
+
+/** @internal */
+export const initializeLogger = ({ pretty, level }: LoggerOptions) => {
+  loggerOptions = { pretty, level };
+  logger = pino(
+    pretty
+      ? {
+          transport: {
+            target: 'pino-pretty',
+            options: {
+              colorize: true,
+            },
+          },
+        }
+      : {},
+  );
+  if (level) {
+    logger.level = level;
+  }
+};

package/src/multimodal/agent_playout.ts

@@ -0,0 +1,254 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { type AudioSource } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import { AudioByteStream } from '../audio.js';
+import type { TranscriptionForwarder } from '../transcription.js';
+import { type AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+
+export const proto = {};
+
+export class PlayoutHandle extends EventEmitter {
+  #audioSource: AudioSource;
+  #sampleRate: number;
+  #itemId: string;
+  #contentIndex: number;
+  /** @internal */
+  transcriptionFwd: TranscriptionForwarder;
+  /** @internal */
+  doneFut: Future;
+  /** @internal */
+  intFut: Future;
+  /** @internal */
+  #interrupted: boolean;
+  /** @internal */
+  pushedDuration: number;
+  /** @internal */
+  totalPlayedTime: number | undefined; // Set when playout is done
+
+  constructor(
+    audioSource: AudioSource,
+    sampleRate: number,
+    itemId: string,
+    contentIndex: number,
+    transcriptionFwd: TranscriptionForwarder,
+  ) {
+    super();
+    this.#audioSource = audioSource;
+    this.#sampleRate = sampleRate;
+    this.#itemId = itemId;
+    this.#contentIndex = contentIndex;
+    this.transcriptionFwd = transcriptionFwd;
+    this.doneFut = new Future();
+    this.intFut = new Future();
+    this.#interrupted = false;
+    this.pushedDuration = 0;
+    this.totalPlayedTime = undefined;
+  }
+
+  get itemId(): string {
+    return this.#itemId;
+  }
+
+  get audioSamples(): number {
+    if (this.totalPlayedTime !== undefined) {
+      return Math.floor(this.totalPlayedTime * this.#sampleRate);
+    }
+
+    return Math.floor(this.pushedDuration - this.#audioSource.queuedDuration * this.#sampleRate);
+  }
+
+  get textChars(): number {
+    return this.transcriptionFwd.currentCharacterIndex;
+  }
+
+  get contentIndex(): number {
+    return this.#contentIndex;
+  }
+
+  get interrupted(): boolean {
+    return this.#interrupted;
+  }
+
+  get done(): boolean {
+    return this.doneFut.done || this.#interrupted;
+  }
+
+  interrupt() {
+    if (this.doneFut.done) return;
+    this.intFut.resolve();
+    this.#interrupted = true;
+  }
+}
+
+export class AgentPlayout {
+  #audioSource: AudioSource;
+  #playoutTask: CancellablePromise<void> | null;
+  #sampleRate: number;
+  #numChannels: number;
+  #inFrameSize: number;
+  #outFrameSize: number;
+  constructor(
+    audioSource: AudioSource,
+    sampleRate: number,
+    numChannels: number,
+    inFrameSize: number,
+    outFrameSize: number,
+  ) {
+    this.#audioSource = audioSource;
+    this.#playoutTask = null;
+    this.#sampleRate = sampleRate;
+    this.#numChannels = numChannels;
+    this.#inFrameSize = inFrameSize;
+    this.#outFrameSize = outFrameSize;
+  }
+
+  play(
+    itemId: string,
+    contentIndex: number,
+    transcriptionFwd: TranscriptionForwarder,
+    textStream: AsyncIterableQueue<string>,
+    audioStream: AsyncIterableQueue<AudioFrame>,
+  ): PlayoutHandle {
+    const handle = new PlayoutHandle(
+      this.#audioSource,
+      this.#sampleRate,
+      itemId,
+      contentIndex,
+      transcriptionFwd,
+    );
+    this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
+    return handle;
+  }
+
+  #makePlayoutTask(
+    oldTask: CancellablePromise<void> | null,
+    handle: PlayoutHandle,
+    textStream: AsyncIterableQueue<string>,
+    audioStream: AsyncIterableQueue<AudioFrame>,
+  ): CancellablePromise<void> {
+    return new CancellablePromise<void>((resolve, reject, onCancel) => {
+      let cancelled = false;
+      onCancel(() => {
+        cancelled = true;
+      });
+
+      (async () => {
+        try {
+          if (oldTask) {
+            await gracefullyCancel(oldTask);
+          }
+
+          let firstFrame = true;
+
+          const readText = () =>
+            new CancellablePromise<void>((resolveText, rejectText, onCancelText) => {
+              let cancelledText = false;
+              onCancelText(() => {
+                cancelledText = true;
+              });
+
+              (async () => {
+                try {
+                  for await (const text of textStream) {
+                    if (cancelledText || cancelled) {
+                      break;
+                    }
+                    handle.transcriptionFwd.pushText(text);
+                  }
+                  resolveText();
+                } catch (error) {
+                  rejectText(error);
+                }
+              })();
+            });
+
+          const capture = () =>
+            new CancellablePromise<void>((resolveCapture, rejectCapture, onCancelCapture) => {
+              let cancelledCapture = false;
+              onCancelCapture(() => {
+                cancelledCapture = true;
+              });
+
+              (async () => {
+                try {
+                  const samplesPerChannel = this.#outFrameSize;
+                  const bstream = new AudioByteStream(
+                    this.#sampleRate,
+                    this.#numChannels,
+                    samplesPerChannel,
+                  );
+
+                  for await (const frame of audioStream) {
+                    if (cancelledCapture || cancelled) {
+                      break;
+                    }
+                    if (firstFrame) {
+                      handle.transcriptionFwd.start();
+                      firstFrame = false;
+                    }
+
+                    handle.transcriptionFwd.pushAudio(frame);
+
+                    for (const f of bstream.write(frame.data.buffer)) {
+                      handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                      await this.#audioSource.captureFrame(f);
+                    }
+                  }
+
+                  if (!cancelledCapture && !cancelled) {
+                    for (const f of bstream.flush()) {
+                      handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                      await this.#audioSource.captureFrame(f);
+                    }
+
+                    handle.transcriptionFwd.markAudioComplete();
+
+                    await this.#audioSource.waitForPlayout();
+                  }
+
+                  resolveCapture();
+                } catch (error) {
+                  rejectCapture(error);
+                }
+              })();
+            });
+
+          const readTextTask = readText();
+          const captureTask = capture();
+
+          try {
+            await Promise.race([captureTask, handle.intFut.await]);
+          } finally {
+            if (!captureTask.isCancelled) {
+              await gracefullyCancel(captureTask);
+            }
+
+            handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+
+            if (handle.interrupted || captureTask.error) {
+              this.#audioSource.clearQueue(); // make sure to remove any queued frames
+            }
+
+            if (!readTextTask.isCancelled) {
+              await gracefullyCancel(readTextTask);
+            }
+
+            if (!firstFrame && !handle.interrupted) {
+              handle.transcriptionFwd.markTextComplete();
+            }
+
+            handle.doneFut.resolve();
+            await handle.transcriptionFwd.close(handle.interrupted);
+          }
+
+          resolve();
+        } catch (error) {
+          reject(error);
+        }
+      })();
+    });
+  }
+}

package/src/multimodal/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './multimodal_agent.js';
+export * from './agent_playout.js';