@livekit/agents 0.1.0

package/src/http_server.ts

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'http';
+
+const healthCheck = async (res: ServerResponse) => {
+  res.writeHead(200);
+  res.end('OK');
+};
+
+export class HTTPServer {
+  host: string;
+  port: number;
+  app: Server;
+
+  constructor(host: string, port: number) {
+    this.host = host;
+    this.port = port;
+
+    this.app = createServer((req: IncomingMessage, res: ServerResponse) => {
+      if (req.url === '/') {
+        healthCheck(res);
+      } else {
+        res.writeHead(404);
+        res.end('not found');
+      }
+    });
+  }
+
+  async run(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.app.listen(this.port, this.host, (err?: Error) => {
+        if (err) reject(err);
+        resolve();
+      });
+    });
+  }
+
+  async close(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.app.close((err?: Error) => {
+        if (err) reject(err);
+        resolve();
+      });
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import * as cli from './cli.js';
+import * as stt from './stt/index.js';
+import * as tts from './tts/index.js';
+
+export * from './vad.js';
+export * from './plugin.js';
+export * from './version.js';
+export * from './job_context.js';
+export * from './job_request.js';
+export * from './worker.js';
+export * from './utils.js';
+export * from './log.js';
+export * from './generator.js';
+
+export { cli, stt, tts };

package/src/ipc/job_main.ts

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type JobAssignment, ServerMessage } from '@livekit/protocol';
+import { Room } from '@livekit/rtc-node';
+import { type ChildProcess, fork } from 'child_process';
+import { EventEmitter, once } from 'events';
+import { JobContext } from '../job_context.js';
+import { log } from '../log.js';
+import { IPC_MESSAGE, type JobMainArgs, type Message, type Ping } from './protocol.js';
+
+export const runJob = (args: JobMainArgs): ChildProcess => {
+  return fork(import.meta.filename, [args.raw, args.entry, args.fallbackURL]);
+};
+
+if (process.send) {
+  // process.argv:
+  //   [0] `node'
+  //   [1] import.meta.filename
+  //   [2] proto.JobAssignment, serialized to JSON string
+  //   [3] import.meta.filename of function containing entry file
+  //   [4] fallback URL in case JobAssignment.url is empty
+
+  const msg = new ServerMessage();
+  msg.fromJsonString(process.argv[2]);
+  const args = msg.message.value as JobAssignment;
+
+  const room = new Room();
+  const closeEvent = new EventEmitter();
+  let shuttingDown = false;
+  let closed = false;
+
+  process.on('message', (msg: Message) => {
+    if (msg.type === IPC_MESSAGE.ShutdownRequest) {
+      shuttingDown = true;
+      closed = true;
+      closeEvent.emit('close');
+    } else if (msg.type === IPC_MESSAGE.Ping) {
+      process.send!({
+        type: IPC_MESSAGE.Pong,
+        lastTimestamp: (msg as Ping).timestamp,
+        timestamp: Date.now(),
+      });
+    }
+  });
+
+  // don't do anything on C-c
+  process.on('SIGINT', () => {});
+
+  const conn = room.connect(args.url || process.argv[4], args.token);
+
+  const start = () => {
+    if (room.isConnected && !closed) {
+      process.send!({ type: IPC_MESSAGE.StartJobResponse });
+
+      // here we import the file containing the exported entry function, and call it.
+      // the file must export default an Agent, usually using defineAgent().
+      import(process.argv[3]).then((agent) => {
+        agent.default.entry(new JobContext(closeEvent, args.job!, room));
+      });
+    }
+  };
+
+  new Promise(() => {
+    conn
+      .then(() => {
+        if (!closed) start();
+      })
+      .catch((err) => {
+        if (!closed) process.send!({ type: IPC_MESSAGE.StartJobResponse, err });
+      });
+  });
+
+  await once(closeEvent, 'close');
+  log.debug('disconnecting from room');
+  await room.disconnect();
+  if (shuttingDown) {
+    process.send({ type: IPC_MESSAGE.ShutdownResponse });
+  } else {
+    process.send({ type: IPC_MESSAGE.UserExit });
+  }
+  process.exit();
+}

package/src/ipc/job_process.ts

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { Job } from '@livekit/protocol';
+import type { ChildProcess } from 'child_process';
+import { once } from 'events';
+import type { Logger } from 'pino';
+import type { AcceptData } from '../job_request.js';
+import { log } from '../log.js';
+import {
+  IPC_MESSAGE,
+  type JobMainArgs,
+  type Message,
+  type Pong,
+  type StartJobResponse,
+} from './protocol.js';
+
+const START_TIMEOUT = 90 * 1000;
+const PING_INTERVAL = 5 * 1000;
+const PING_TIMEOUT = 90 * 1000;
+const HIGH_PING_THRESHOLD = 10;
+
+export class JobProcess {
+  #job: Job;
+  args: JobMainArgs;
+  logger: Logger;
+  process?: ChildProcess;
+  startTimeout?: ReturnType<typeof setTimeout>;
+  pongTimeout?: ReturnType<typeof setTimeout>;
+  pingInterval?: ReturnType<typeof setInterval>;
+
+  constructor(job: Job, acceptData: AcceptData, raw: string, fallbackURL: string) {
+    this.#job = job;
+    this.args = { entry: acceptData.entry, raw, fallbackURL };
+    this.logger = log.child({ job_id: this.#job.id });
+  }
+
+  get job(): Job {
+    return this.#job;
+  }
+
+  async close() {
+    this.logger.debug('closing job process');
+    await this.clear();
+    this.process!.send({ type: IPC_MESSAGE.ShutdownRequest });
+    await once(this.process!, 'disconnect');
+    this.logger.info('job process closed');
+  }
+
+  async clear() {
+    clearTimeout(this.startTimeout);
+    clearTimeout(this.pongTimeout);
+    clearInterval(this.pingInterval);
+  }
+
+  async run() {
+    let resp: StartJobResponse | undefined = undefined;
+
+    this.startTimeout = setTimeout(() => {
+      if (resp === undefined) {
+        this.logger.error('process start timed out, killing job');
+        this.process?.kill();
+        this.clear();
+      }
+    }, START_TIMEOUT);
+
+    this.pingInterval = setInterval(() => {
+      this.process?.send({ type: IPC_MESSAGE.Ping, timestamp: Date.now() });
+    }, PING_INTERVAL);
+
+    this.pongTimeout = setTimeout(() => {
+      this.logger.error('job ping timed out, killing job');
+      this.process?.kill();
+      this.clear();
+    }, PING_TIMEOUT);
+
+    this.process = await import('./job_main.js').then((main) => main.runJob(this.args));
+
+    this.process.on('message', (msg: Message) => {
+      if (msg.type === IPC_MESSAGE.StartJobResponse) {
+        resp = msg as StartJobResponse;
+      } else if (msg.type === IPC_MESSAGE.Pong) {
+        const delay = Date.now() - (msg as Pong).timestamp;
+        if (delay > HIGH_PING_THRESHOLD) {
+          this.logger.warn(`job is unresponsive (${delay}ms delay)`);
+        }
+        this.pongTimeout?.refresh();
+      } else if (msg.type === IPC_MESSAGE.UserExit || msg.type === IPC_MESSAGE.ShutdownResponse) {
+        this.logger.debug('job exiting');
+        this.clear();
+      }
+    });
+
+    await once(this.process, 'disconnect');
+  }
+}

package/src/ipc/protocol.ts

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type JobMainArgs = {
+  entry: string;
+  raw: string;
+  fallbackURL: string;
+};
+
+export interface Message {
+  type: IPC_MESSAGE;
+}
+
+export interface StartJobResponse extends Message {
+  type: IPC_MESSAGE.StartJobResponse;
+  err?: Error;
+}
+
+export interface Ping extends Message {
+  type: IPC_MESSAGE.Ping;
+  timestamp: number;
+}
+
+export interface Pong extends Message {
+  type: IPC_MESSAGE.Pong;
+  lastTimestamp: number;
+  timestamp: number;
+}
+
+export interface ShutdownRequest extends Message {
+  type: IPC_MESSAGE.ShutdownRequest;
+}
+
+export interface ShutdownResponse extends Message {
+  type: IPC_MESSAGE.ShutdownResponse;
+}
+
+export interface UserExit extends Message {
+  type: IPC_MESSAGE.UserExit;
+}
+
+export enum IPC_MESSAGE {
+  StartJobRequest,
+  StartJobResponse,
+  Ping,
+  Pong,
+  ShutdownRequest,
+  ShutdownResponse,
+  UserExit,
+}

package/src/job_context.ts

@@ -0,0 +1,49 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { Job } from '@livekit/protocol';
+import type { LocalParticipant, RemoteParticipant, Room } from '@livekit/rtc-node';
+import type { EventEmitter } from 'events';
+
+export class JobContext {
+  #job: Job;
+  #room: Room;
+  #publisher?: RemoteParticipant;
+  tx: EventEmitter;
+
+  constructor(
+    tx: EventEmitter,
+    job: Job,
+    room: Room,
+    publisher: RemoteParticipant | undefined = undefined,
+  ) {
+    this.#job = job;
+    this.#room = room;
+    this.#publisher = publisher;
+    this.tx = tx;
+  }
+
+  get id(): string {
+    return this.#job.id;
+  }
+
+  get job(): Job {
+    return this.#job;
+  }
+
+  get room(): Room {
+    return this.#room;
+  }
+
+  get publisher(): RemoteParticipant | undefined {
+    return this.#publisher;
+  }
+
+  get agent(): LocalParticipant | undefined {
+    return this.#room.localParticipant;
+  }
+
+  async shutdown() {
+    this.tx.emit('close');
+  }
+}

package/src/job_request.ts

@@ -0,0 +1,118 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { Job, ParticipantInfo, Room } from '@livekit/protocol';
+import { EventEmitter } from 'events';
+import { log } from './log.js';
+
+class AnsweredError extends Error {
+  constructor() {
+    super();
+    this.name = 'AnsweredError';
+    this.message = 'request already answered';
+  }
+}
+
+enum AutoDisconnect {
+  ROOM_EMPTY,
+  PUBLISHER_LEFT,
+  NONE,
+}
+
+export enum AutoSubscribe {
+  SUBSCRIBE_ALL,
+  SUBSCRIBE_NONE,
+  VIDEO_ONLY,
+  AUDIO_ONLY,
+}
+
+export type AcceptData = {
+  entry: string; // filename
+  autoSubscribe: AutoSubscribe;
+  autoDisconnect: AutoDisconnect;
+  name: string;
+  identity: string;
+  metadata: string;
+  assign: EventEmitter;
+};
+
+export type AvailRes = {
+  avail: boolean;
+  data?: AcceptData;
+};
+
+export class JobRequest {
+  #job: Job;
+  #answered = false;
+  tx: EventEmitter;
+  logger = log.child({ job: this.job });
+
+  constructor(job: Job, tx: EventEmitter) {
+    this.#job = job;
+    this.tx = tx;
+  }
+
+  get id(): string {
+    return this.#job.id;
+  }
+
+  get job(): Job {
+    return this.#job;
+  }
+
+  get room(): Room | undefined {
+    return this.#job.room;
+  }
+
+  get publisher(): ParticipantInfo | undefined {
+    return this.#job.participant;
+  }
+
+  get answered(): boolean {
+    return this.#answered;
+  }
+
+  async reject() {
+    if (this.#answered) {
+      throw new AnsweredError();
+    }
+    this.#answered = true;
+    this.tx.emit('recv', { avail: false, data: undefined } as AvailRes);
+    this.logger.info('rejected job', this.id);
+  }
+
+  async accept(
+    entry: string,
+    autoSubscribe: AutoSubscribe = AutoSubscribe.SUBSCRIBE_ALL,
+    autoDisconnect: AutoDisconnect = AutoDisconnect.ROOM_EMPTY,
+    name: string = '',
+    identity: string = '',
+    metadata: string = '',
+  ) {
+    if (this.#answered) {
+      throw new AnsweredError();
+    }
+    this.#answered = true;
+
+    const assign = new EventEmitter();
+    assign.on('error', (e) => {
+      throw e;
+    });
+
+    if (identity === '') identity = 'agent-' + this.id;
+
+    const data: AcceptData = {
+      entry,
+      autoSubscribe,
+      autoDisconnect,
+      name,
+      identity,
+      metadata,
+      assign,
+    };
+
+    this.tx.emit('recv', { avail: true, data } as AvailRes);
+
+    this.logger.info('accepted job', this.id);
+  }
+}

package/src/log.ts

@@ -0,0 +1,13 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { pino } from 'pino';
+
+export const log = pino({
+  transport: {
+    target: 'pino-pretty',
+    options: {
+      colorize: true,
+    },
+  },
+});

package/src/plugin.ts

@@ -0,0 +1,28 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export abstract class Plugin {
+  registeredPlugins: Plugin[] = [];
+  #title: string;
+  #version: string;
+
+  constructor(title: string, version: string) {
+    this.#title = title;
+    this.#version = version;
+  }
+
+  public static registerPlugins(plugin: Plugin) {
+    plugin.registeredPlugins.push(plugin);
+  }
+
+  abstract downloadFiles(): void;
+
+  get title(): string {
+    return this.#title;
+  }
+
+  get version(): string {
+    return this.#version;
+  }
+}

package/src/stt/index.ts

@@ -0,0 +1,6 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export { STT, SpeechEvent, SpeechEventType, SpeechStream } from './stt.js';
+export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';

package/src/stt/stream_adapter.ts

@@ -0,0 +1,104 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { type AudioBuffer, mergeFrames } from '../utils.js';
+import { VADEventType, type VADStream } from '../vad.js';
+import { STT, SpeechEvent, SpeechEventType, SpeechStream } from './stt.js';
+
+export class StreamAdapterWrapper extends SpeechStream {
+  closed: boolean;
+  stt: STT;
+  vadStream: VADStream;
+  eventQueue: (SpeechEvent | undefined)[];
+  language?: string;
+  task: {
+    run: Promise<void>;
+    cancel: () => void;
+  };
+
+  constructor(stt: STT, vadStream: VADStream, language: string | undefined = undefined) {
+    super();
+    this.closed = false;
+    this.stt = stt;
+    this.vadStream = vadStream;
+    this.eventQueue = [];
+    this.language = language;
+    this.task = {
+      run: new Promise((_, reject) => {
+        this.run(reject);
+      }),
+      cancel: () => {},
+    };
+  }
+
+  async run(reject: (arg: Error) => void) {
+    this.task.cancel = () => {
+      this.closed = true;
+      reject(new Error('cancelled'));
+    };
+
+    for (const event of this.vadStream) {
+      if (event.type == VADEventType.START_OF_SPEECH) {
+        const startEvent = new SpeechEvent(SpeechEventType.START_OF_SPEECH);
+        this.eventQueue.push(startEvent);
+      } else if (event.type == VADEventType.END_OF_SPEECH) {
+        const mergedFrames = mergeFrames(event.speech);
+        const endEvent = await this.stt.recognize(mergedFrames, this.language);
+        this.eventQueue.push(endEvent);
+      }
+    }
+
+    this.eventQueue.push(undefined);
+  }
+
+  pushFrame(frame: AudioFrame) {
+    if (this.closed) {
+      throw new TypeError('cannot push frame to closed stream');
+    }
+
+    this.vadStream.pushFrame(frame);
+  }
+
+  async close(wait: boolean = true): Promise<void> {
+    this.closed = true;
+
+    if (!wait) {
+      this.task.cancel();
+    }
+
+    await this.vadStream.close(wait);
+    await this.task.run;
+  }
+
+  next(): IteratorResult<SpeechEvent> {
+    const item = this.eventQueue.shift();
+    if (item) {
+      return { done: false, value: item };
+    } else {
+      return { done: true, value: undefined };
+    }
+  }
+}
+
+export class StreamAdapter extends STT {
+  stt: STT;
+  vadStream: VADStream;
+
+  constructor(stt: STT, vadStream: VADStream) {
+    super(true);
+    this.stt = stt;
+    this.vadStream = vadStream;
+  }
+
+  async recognize(
+    buffer: AudioBuffer,
+    language: string | undefined = undefined,
+  ): Promise<SpeechEvent> {
+    return await this.stt.recognize(buffer, language);
+  }
+
+  stream(language: string | undefined = undefined) {
+    return new StreamAdapterWrapper(this.stt, this.vadStream, language);
+  }
+}

package/src/stt/stt.ts

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import type { AudioBuffer } from '../utils.js';
+
+export enum SpeechEventType {
+  START_OF_SPEECH = 0,
+  INTERIM_TRANSCRIPT = 1,
+  FINAL_TRANSCRIPT = 2,
+  END_OF_SPEECH = 3,
+}
+
+export interface SpeechData {
+  language: string;
+  text: string;
+  startTime: number;
+  endTime: number;
+  confidence: number;
+}
+
+export class SpeechEvent {
+  type: SpeechEventType;
+  alternatives: SpeechData[];
+
+  constructor(type: SpeechEventType, alternatives: SpeechData[] = []) {
+    this.type = type;
+    this.alternatives = alternatives;
+  }
+}
+
+export abstract class SpeechStream implements IterableIterator<SpeechEvent> {
+  abstract pushFrame(token: AudioFrame): void;
+
+  abstract close(wait: boolean): Promise<void>;
+
+  abstract next(): IteratorResult<SpeechEvent>;
+
+  [Symbol.iterator](): SpeechStream {
+    return this;
+  }
+}
+
+export abstract class STT {
+  #streamingSupported: boolean;
+
+  constructor(streamingSupported: boolean) {
+    this.#streamingSupported = streamingSupported;
+  }
+
+  abstract recognize(buffer: AudioBuffer, language?: string): Promise<SpeechEvent>;
+
+  abstract stream(language: string | undefined): SpeechStream;
+
+  get streamingSupported(): boolean {
+    return this.#streamingSupported;
+  }
+}

package/src/tokenize.ts

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export interface SegmentedSentence {
+  text: string;
+}
+
+export abstract class SentenceTokenizer {
+  abstract tokenize(text: string, language?: string): SegmentedSentence[];
+  abstract stream(language: string | undefined): SentenceStream;
+}
+
+export abstract class SentenceStream implements IterableIterator<SegmentedSentence> {
+  abstract pushText(text: string): void;
+  abstract flush(): Promise<void>;
+  async close(): Promise<void> {}
+  abstract next(): IteratorResult<SegmentedSentence>;
+  [Symbol.iterator](): SentenceStream {
+    return this;
+  }
+}