@livekit/agents 0.1.0 → 0.2.0

package/src/ipc/proc_job_executor.ts

@@ -0,0 +1,162 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChildProcess } from 'child_process';
+import { once } from 'events';
+import type { RunningJobInfo } from '../job.js';
+import { log, loggerOptions } from '../log.js';
+import { Future } from '../utils.js';
+import type { ProcOpts } from './job_executor.js';
+import { JobExecutor } from './job_executor.js';
+import type { IPCMessage } from './message.js';
+
+export class ProcJobExecutor extends JobExecutor {
+  #opts: ProcOpts;
+  #started = false;
+  #closing = false;
+  #runningJob?: RunningJobInfo = undefined;
+  #proc?: ChildProcess;
+  #pingInterval?: ReturnType<typeof setInterval>;
+  #pongTimeout?: ReturnType<typeof setTimeout>;
+  #init = new Future();
+  #join = new Future();
+  #logger = log().child({ runningJob: this.#runningJob });
+
+  constructor(agent: string, initializeTimeout: number, closeTimeout: number) {
+    super();
+    this.#opts = {
+      agent,
+      initializeTimeout,
+      closeTimeout,
+    };
+  }
+
+  get started(): boolean {
+    return this.#started;
+  }
+
+  get runningJob(): RunningJobInfo | undefined {
+    return this.#runningJob;
+  }
+
+  async start() {
+    if (this.#started) {
+      throw new Error('runner already started');
+    } else if (this.#closing) {
+      throw new Error('runner is closed');
+    }
+
+    this.#proc = await import('./job_main.js').then((m) =>
+      m.runProcess({
+        agentFile: this.#opts.agent,
+      }),
+    );
+
+    this.#started = true;
+    this.run();
+  }
+
+  async run() {
+    await this.#init.await;
+
+    this.#pingInterval = setInterval(() => {
+      this.#proc!.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
+    }, this.PING_INTERVAL);
+
+    this.#pongTimeout = setTimeout(() => {
+      this.#logger.warn('job is unresponsive');
+      clearTimeout(this.#pongTimeout);
+      clearInterval(this.#pingInterval);
+      this.#proc!.kill();
+      this.#join.resolve();
+    }, this.PING_TIMEOUT);
+
+    const listener = (msg: IPCMessage) => {
+      switch (msg.case) {
+        case 'pongResponse': {
+          const delay = Date.now() - msg.value.timestamp;
+          if (delay > this.HIGH_PING_THRESHOLD) {
+            this.#logger.child({ delay }).warn('job executor is unresponsive');
+          }
+          this.#pongTimeout?.refresh();
+          break;
+        }
+        case 'exiting': {
+          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');
+          break;
+        }
+        case 'done': {
+          this.#closing = true;
+          this.#proc!.off('message', listener);
+          this.#join.resolve();
+          break;
+        }
+      }
+    };
+    this.#proc!.on('message', listener);
+    this.#proc!.on('error', (err) => {
+      if (this.#closing) return;
+      this.#logger.child({ err }).warn('job process exited unexpectedly');
+      clearTimeout(this.#pongTimeout);
+      clearInterval(this.#pingInterval);
+      this.#join.resolve();
+    });
+
+    await this.#join.await;
+  }
+
+  async join() {
+    if (!this.#started) {
+      throw new Error('runner not started');
+    }
+
+    await this.#join.await;
+  }
+
+  async initialize() {
+    const timer = setTimeout(() => {
+      const err = new Error('runner initialization timed out');
+      this.#init.reject(err);
+      throw err;
+    }, this.#opts.initializeTimeout);
+    this.#proc!.send({ case: 'initializeRequest', value: { loggerOptions } });
+    await once(this.#proc!, 'message').then(([msg]: IPCMessage[]) => {
+      clearTimeout(timer);
+      if (msg.case !== 'initializeResponse') {
+        throw new Error('first message must be InitializeResponse');
+      }
+    });
+    this.#init.resolve();
+  }
+
+  async close() {
+    if (!this.#started) {
+      return;
+    }
+    this.#closing = true;
+
+    if (!this.#runningJob) {
+      this.#proc!.kill();
+      this.#join.resolve();
+    }
+
+    this.#proc!.send({ case: 'shutdownRequest' });
+
+    const timer = setTimeout(() => {
+      this.#logger.error('job shutdown is taking too much time');
+    }, this.#opts.closeTimeout);
+    await this.#join.await.then(() => {
+      clearTimeout(timer);
+      clearTimeout(this.#pongTimeout);
+      clearInterval(this.#pingInterval);
+    });
+  }
+
+  async launchJob(info: RunningJobInfo) {
+    if (this.#runningJob) {
+      throw new Error('executor already has a running job');
+    }
+    this.#runningJob = info;
+    this.#proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
+  }
+}

package/src/ipc/proc_pool.ts

@@ -0,0 +1,108 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { RunningJobInfo } from '../job.js';
+import { Mutex, Queue } from '../utils.js';
+import type { JobExecutor } from './job_executor.js';
+import { ProcJobExecutor } from './proc_job_executor.js';
+
+export class ProcPool {
+  agent: string;
+  initializeTimeout: number;
+  closeTimeout: number;
+  executors: JobExecutor[] = [];
+  tasks: Promise<void>[] = [];
+  started = false;
+  closed = false;
+  controller = new AbortController();
+  initMutex = new Mutex();
+  procMutex: Mutex;
+  procUnlock?: () => void;
+  warmedProcQueue = new Queue<JobExecutor>();
+
+  constructor(
+    agent: string,
+    numIdleProcesses: number,
+    initializeTimeout: number,
+    closeTimeout: number,
+  ) {
+    this.agent = agent;
+    this.procMutex = new Mutex(numIdleProcesses);
+    this.initializeTimeout = initializeTimeout;
+    this.closeTimeout = closeTimeout;
+  }
+
+  get processes(): JobExecutor[] {
+    return this.executors;
+  }
+
+  getByJobId(id: string): JobExecutor | null {
+    return this.executors.find((x) => x.runningJob && x.runningJob.job.id === id) || null;
+  }
+
+  async launchJob(info: RunningJobInfo) {
+    const proc = await this.warmedProcQueue.get();
+    if (this.procUnlock) {
+      this.procUnlock();
+      this.procUnlock = undefined;
+    }
+    await proc.launchJob(info);
+  }
+
+  async procWatchTask() {
+    const proc = new ProcJobExecutor(this.agent, this.initializeTimeout, this.closeTimeout);
+
+    try {
+      this.executors.push(proc);
+
+      const unlock = await this.initMutex.lock();
+      if (this.closed) {
+        return;
+      }
+
+      await proc.start();
+      try {
+        await proc.initialize();
+        await this.warmedProcQueue.put(proc);
+      } catch {
+        if (this.procUnlock) {
+          this.procUnlock();
+          this.procUnlock = undefined;
+        }
+      }
+
+      unlock();
+      await proc.join();
+    } finally {
+      this.executors.splice(this.executors.indexOf(proc));
+    }
+  }
+
+  start() {
+    if (this.started) {
+      return;
+    }
+
+    this.started = true;
+    this.run(this.controller.signal);
+  }
+
+  async run(signal: AbortSignal) {
+    while (!signal.aborted) {
+      this.procUnlock = await this.procMutex.lock();
+      const task = this.procWatchTask();
+      this.tasks.push(task);
+      task.finally(() => this.tasks.splice(this.tasks.indexOf(task)));
+    }
+  }
+
+  async close() {
+    if (!this.started) {
+      return;
+    }
+    this.closed = true;
+    this.controller.abort();
+    this.executors.forEach((e) => e.close());
+    await Promise.allSettled(this.tasks);
+  }
+}

package/src/job.ts

@@ -0,0 +1,258 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type * as proto from '@livekit/protocol';
+import type {
+  E2EEOptions,
+  LocalParticipant,
+  RemoteParticipant,
+  Room,
+  RtcConfiguration,
+} from '@livekit/rtc-node';
+import { RoomEvent, TrackKind } from '@livekit/rtc-node';
+import type { Logger } from 'pino';
+import { log } from './log.js';
+
+/** Which tracks, if any, should the agent automatically subscribe to? */
+export enum AutoSubscribe {
+  SUBSCRIBE_ALL,
+  SUBSCRIBE_NONE,
+  VIDEO_ONLY,
+  AUDIO_ONLY,
+}
+
+export type JobAcceptArguments = {
+  name: string;
+  identity: string;
+  metadata: string;
+};
+
+export type RunningJobInfo = {
+  acceptArguments: JobAcceptArguments;
+  job: proto.Job;
+  url: string;
+  token: string;
+};
+
+/** Attempted to add a function callback, but the function already exists. */
+export class FunctionExistsError extends Error {
+  constructor(msg?: string) {
+    super(msg);
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+}
+
+/** The job and environment context as seen by the agent, accessible by the entrypoint function. */
+export class JobContext {
+  #proc: JobProcess;
+  #info: RunningJobInfo;
+  #room: Room;
+  #onConnect: () => void;
+  #onShutdown: (s: string) => void;
+  /** @internal */
+  shutdownCallbacks: (() => Promise<void>)[] = [];
+  #participantEntrypoints: ((job: JobContext, p: RemoteParticipant) => Promise<void>)[] = [];
+  #participantTasks: {
+    [id: string]: {
+      callback: (job: JobContext, p: RemoteParticipant) => Promise<void>;
+      result: Promise<void>;
+    };
+  } = {};
+  #logger: Logger;
+
+  constructor(
+    proc: JobProcess,
+    info: RunningJobInfo,
+    room: Room,
+    onConnect: () => void,
+    onShutdown: (s: string) => void,
+  ) {
+    this.#proc = proc;
+    this.#info = info;
+    this.#room = room;
+    this.#onConnect = onConnect;
+    this.#onShutdown = onShutdown;
+    this.onParticipantConnected = this.onParticipantConnected.bind(this);
+    this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
+    this.#logger = log().child({ info: this.#info });
+  }
+
+  get proc(): JobProcess {
+    return this.#proc;
+  }
+
+  get job(): proto.Job {
+    return this.#info.job;
+  }
+
+  /** @returns The room the agent was called into */
+  get room(): Room {
+    return this.#room;
+  }
+
+  /** @returns The agent's participant if connected to the room, otherwise `undefined` */
+  get agent(): LocalParticipant | undefined {
+    return this.#room.localParticipant;
+  }
+
+  /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */
+  addShutdownCallback(callback: () => Promise<void>) {
+    this.shutdownCallbacks.push(callback);
+  }
+
+  /**
+   * Connects the agent to the room.
+   *
+   * @remarks
+   * It is recommended to run this command as early in the function as possible, as executing it
+   * later may cause noticeable delay between user and agent joins.
+   *
+   * @see {@link https://github.com/livekit/node-sdks/tree/main/packages/livekit-rtc#readme |
+   * @livekit/rtc-node} for more information about the parameters.
+   */
+  async connect(
+    e2ee?: E2EEOptions,
+    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,
+    rtcConfig?: RtcConfiguration,
+  ) {
+    const opts = {
+      e2ee,
+      autoSubscribe: autoSubscribe == AutoSubscribe.SUBSCRIBE_ALL,
+      rtcConfig,
+      dynacast: false,
+    };
+
+    await this.#room.connect(this.#info.url, this.#info.token, opts);
+    this.#onConnect();
+
+    this.#room.remoteParticipants.forEach(this.onParticipantConnected);
+
+    if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {
+      this.#room.remoteParticipants.forEach((p) => {
+        p.trackPublications.forEach((pub) => {
+          if (
+            (autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||
+            (autoSubscribe === AutoSubscribe.VIDEO_ONLY && pub.kind === TrackKind.KIND_VIDEO)
+          ) {
+            pub.setSubscribed(true);
+          }
+        });
+      });
+    }
+  }
+
+  /**
+   * Gracefully shuts down the job, and runs all shutdown promises.
+   *
+   * @param reason - Optional reason for shutdown
+   */
+  shutdown(reason = '') {
+    this.#onShutdown(reason);
+  }
+
+  /** @internal */
+  onParticipantConnected(p: RemoteParticipant) {
+    for (const callback of this.#participantEntrypoints) {
+      if (
+        p.identity in this.#participantTasks &&
+        this.#participantTasks[p.identity].callback == callback
+      ) {
+        this.#logger.warn(
+          'a participant has joined before a prior prticipant task matching the same identity has finished:',
+          p.identity,
+        );
+      }
+      const result = callback(this, p);
+      result.finally(() => delete this.#participantTasks[p.identity]);
+      this.#participantTasks[p.identity] = { callback, result };
+    }
+  }
+
+  /**
+   * Adds a promise to be awaited whenever a new participant joins the room.
+   *
+   * @throws {@link FunctionExistsError} if an entrypoint already exists
+   */
+  addParticipantEntrypoint(callback: (job: JobContext, p: RemoteParticipant) => Promise<void>) {
+    if (this.#participantEntrypoints.includes(callback)) {
+      throw new FunctionExistsError('entrypoints cannot be added more than once');
+    }
+
+    this.#participantEntrypoints.push(callback);
+  }
+}
+
+export class JobProcess {
+  #pid = process.pid;
+  #userData: { [id: string]: unknown } = {};
+
+  get pid(): number {
+    return this.#pid;
+  }
+
+  get userData(): { [id: string]: unknown } {
+    return this.#userData;
+  }
+}
+
+/**
+ * A request sent by the server to spawn a new agent job.
+ *
+ * @remarks
+ * For most applications, this is best left to the default, which simply accepts the job and
+ * handles the logic inside the entrypoint function. This class is useful for vetting which
+ * requests should fill idle processes and which should be outright rejected.
+ */
+export class JobRequest {
+  #job: proto.Job;
+  #onReject: () => Promise<void>;
+  #onAccept: (args: JobAcceptArguments) => Promise<void>;
+
+  /** @internal */
+  constructor(
+    job: proto.Job,
+    onReject: () => Promise<void>,
+    onAccept: (args: JobAcceptArguments) => Promise<void>,
+  ) {
+    this.#job = job;
+    this.#onReject = onReject;
+    this.#onAccept = onAccept;
+  }
+
+  /** @returns The ID of the job, set by the LiveKit server */
+  get id(): string {
+    return this.#job.id;
+  }
+
+  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+  get job(): proto.Job {
+    return this.#job;
+  }
+
+  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+  get room(): proto.Room | undefined {
+    return this.#job.room;
+  }
+
+  /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
+  get publisher(): proto.ParticipantInfo | undefined {
+    return this.#job.participant;
+  }
+
+  /** @returns The agent's name, as set in {@link WorkerOptions} */
+  get agentName(): string {
+    return this.#job.agentName;
+  }
+
+  /** Rejects the job. */
+  async reject() {
+    await this.#onReject();
+  }
+
+  /** Accepts the job, launching it on an idle child process. */
+  async accept(name = '', identity = '', metadata = '') {
+    if (identity === '') identity = 'agent-' + this.id;
+
+    this.#onAccept({ name, identity, metadata });
+  }
+}

package/src/llm/function_context.ts

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { z } from 'zod';
+
+// heavily inspired by Vercel AI's `tool()`:
+// https://github.com/vercel/ai/blob/3b0983b/packages/ai/core/tool/tool.ts
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+/** Type reinforcement for the callable function's execute parameters. */
+export type inferParameters<P extends z.ZodTypeAny> = z.infer<P>;
+
+/** A definition for a function callable by the LLM. */
+export interface CallableFunction<P extends z.ZodTypeAny = any, R = any> {
+  description: string;
+  parameters: P;
+  execute: (args: inferParameters<P>) => PromiseLike<R>;
+}
+
+/** An object containing callable functions and their names */
+export type FunctionContext = {
+  [name: string]: CallableFunction;
+};
+
+/** @internal */
+export const oaiParams = (p: z.AnyZodObject) => {
+  const properties: Record<string, any> = {};
+  const required_properties: string[] = [];
+
+  for (const key in p.shape) {
+    const field = p.shape[key];
+    const description = field._def.description || undefined;
+    let type: string;
+    let enumValues: any[] | undefined;
+
+    if (field instanceof z.ZodEnum) {
+      enumValues = field._def.values;
+      type = typeof enumValues![0];
+    } else {
+      type = field._def.typeName.toLowerCase();
+    }
+
+    properties[key] = {
+      type: type.includes('zod') ? type.substring(3) : type,
+      description,
+      enum: enumValues,
+    };
+
+    if (!field._def.defaultValue) {
+      required_properties.push(key);
+    }
+  }
+
+  const type = 'object' as const;
+  return {
+    type,
+    properties,
+    required_properties,
+  };
+};

package/src/llm/index.ts

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type CallableFunction,
+  type FunctionContext,
+  type inferParameters,
+  oaiParams,
+} from './function_context.js';
+
+export { CallableFunction, FunctionContext, inferParameters, oaiParams };

package/src/log.ts

@@ -1,13 +1,45 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { Logger } from 'pino';
 import { pino } from 'pino';
 
-export const log = pino({
-  transport: {
-    target: 'pino-pretty',
-    options: {
-      colorize: true,
-    },
-  },
-});
+/** @internal */
+export type LoggerOptions = {
+  pretty: boolean;
+  level?: string;
+};
+
+/** @internal */
+export let loggerOptions: LoggerOptions;
+
+/** @internal */
+let logger: Logger | undefined = undefined;
+
+/** @internal */
+export const log = () => {
+  if (!logger) {
+    throw new TypeError('logger not initialized. did you forget to run initializeLogger()?');
+  }
+  return logger;
+};
+
+/** @internal */
+export const initializeLogger = ({ pretty, level }: LoggerOptions) => {
+  loggerOptions = { pretty, level };
+  logger = pino(
+    pretty
+      ? {
+          transport: {
+            target: 'pino-pretty',
+            options: {
+              colorize: true,
+            },
+          },
+        }
+      : {},
+  );
+  if (level) {
+    logger.level = level;
+  }
+};

package/src/stt/index.ts

@@ -2,5 +2,5 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-export { STT, SpeechEvent, SpeechEventType, SpeechStream } from './stt.js';
+export { STT, SpeechEvent, SpeechEventType, SpeechStream, type SpeechData } from './stt.js';
 export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';