@livekit/agents 0.3.4 → 0.4.0

package/src/llm/index.ts

@@ -1,11 +1,30 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import {
+export {
   type CallableFunction,
+  type FunctionCallInfo,
+  type CallableFunctionResult,
   type FunctionContext,
   type inferParameters,
   oaiParams,
+  oaiBuildFunctionInfo,
 } from './function_context.js';
 
-export { CallableFunction, FunctionContext, inferParameters, oaiParams };
+export {
+  type ChatImage,
+  type ChatAudio,
+  type ChatContent,
+  ChatRole,
+  ChatMessage,
+  ChatContext,
+} from './chat_context.js';
+
+export {
+  type ChoiceDelta,
+  type CompletionUsage,
+  type Choice,
+  type ChatChunk,
+  LLM,
+  LLMStream,
+} from './llm.js';

package/src/llm/llm.ts

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { AsyncIterableQueue } from '../utils.js';
+import type { ChatContext, ChatRole } from './chat_context.js';
+import type { FunctionCallInfo, FunctionContext } from './function_context.js';
+
+export interface ChoiceDelta {
+  role: ChatRole;
+  content?: string;
+  toolCalls?: FunctionCallInfo[];
+}
+
+export interface CompletionUsage {
+  completionTokens: number;
+  promptTokens: number;
+  totalTokens: number;
+}
+
+export interface Choice {
+  delta: ChoiceDelta;
+  index: number;
+}
+
+export interface ChatChunk {
+  requestId: string;
+  choices: Choice[];
+  usage?: CompletionUsage;
+}
+
+export abstract class LLM {
+  /**
+   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
+   */
+  abstract chat({
+    chatCtx,
+    fncCtx,
+    temperature,
+    n,
+    parallelToolCalls,
+  }: {
+    chatCtx: ChatContext;
+    fncCtx?: FunctionContext;
+    temperature?: number;
+    n?: number;
+    parallelToolCalls?: boolean;
+  }): LLMStream;
+}
+
+export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+  protected queue = new AsyncIterableQueue<ChatChunk>();
+  protected closed = false;
+  protected _functionCalls: FunctionCallInfo[] = [];
+
+  #chatCtx: ChatContext;
+  #fncCtx?: FunctionContext;
+
+  constructor(chatCtx: ChatContext, fncCtx?: FunctionContext) {
+    this.#chatCtx = chatCtx;
+    this.#fncCtx = fncCtx;
+  }
+
+  /** List of called functions from this stream. */
+  get functionCalls(): FunctionCallInfo[] {
+    return this._functionCalls;
+  }
+
+  /** The function context of this stream. */
+  get fncCtx(): FunctionContext | undefined {
+    return this.#fncCtx;
+  }
+
+  /** The initial chat context of this stream. */
+  get chatCtx(): ChatContext {
+    return this.#chatCtx;
+  }
+
+  /** Execute all deferred functions of this stream concurrently. */
+  executeFunctions(): FunctionCallInfo[] {
+    this._functionCalls.forEach(
+      (f) =>
+        (f.task = f.func.execute(f.params).then(
+          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),
+          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),
+        )),
+    );
+    return this._functionCalls;
+  }
+
+  next(): Promise<IteratorResult<ChatChunk>> {
+    return this.queue.next();
+  }
+
+  close() {
+    this.queue.close();
+    this.closed = true;
+  }
+
+  [Symbol.asyncIterator](): LLMStream {
+    return this;
+  }
+}

package/src/multimodal/multimodal_agent.ts

@@ -64,13 +64,16 @@ export class MultimodalAgent extends EventEmitter {
 
   constructor({
     model,
+    chatCtx,
     fncCtx,
   }: {
     model: RealtimeModel;
-    fncCtx?: llm.FunctionContext | undefined;
+    chatCtx?: llm.ChatContext;
+    fncCtx?: llm.FunctionContext;
   }) {
     super();
     this.model = model;
+    this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
   }
 
@@ -83,6 +86,7 @@ export class MultimodalAgent extends EventEmitter {
   #logger = log();
   #session: RealtimeSession | null = null;
   #fncCtx: llm.FunctionContext | undefined = undefined;
+  #chatCtx: llm.ChatContext | undefined = undefined;
 
   #_started: boolean = false;
   #_pendingFunctionCalls: Set<string> = new Set();
@@ -143,10 +147,19 @@ export class MultimodalAgent extends EventEmitter {
         }
         this.#linkParticipant(participant.identity);
       });
-      room.on(RoomEvent.TrackPublished, () => {
-        // in case we are connected before the participant has published, we'd need to re-subscribe
-        this.#subscribeToMicrophone();
-      });
+      room.on(
+        RoomEvent.TrackPublished,
+        (trackPublication: RemoteTrackPublication, participant: RemoteParticipant) => {
+          if (
+            this.linkedParticipant &&
+            participant.identity === this.linkedParticipant.identity &&
+            trackPublication.source === TrackSource.SOURCE_MICROPHONE &&
+            !trackPublication.subscribed
+          ) {
+            trackPublication.setSubscribed(true);
+          }
+        },
+      );
       room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));
 
       this.room = room;
@@ -200,7 +213,7 @@ export class MultimodalAgent extends EventEmitter {
         }
       }
 
-      this.#session = this.model.session({ fncCtx: this.#fncCtx });
+      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });
       this.#started = true;
 
       // eslint-disable-next-line @typescript-eslint/no-explicit-any

package/src/pipeline/agent_output.ts

@@ -0,0 +1,185 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { log } from '../log.js';
+import { SynthesizeStream, type TTS } from '../tts/index.js';
+import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+import type { AgentPlayout, PlayoutHandle } from './agent_playout.js';
+
+export type SpeechSource = AsyncIterable<string> | string | Promise<string>;
+
+export class SynthesisHandle {
+  static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+
+  #speechId: string;
+  ttsSource: SpeechSource;
+  #agentPlayout: AgentPlayout;
+  tts: TTS;
+  queue = new AsyncIterableQueue<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>();
+  #playHandle?: PlayoutHandle;
+  intFut = new Future();
+  #logger = log();
+
+  constructor(speechId: string, ttsSource: SpeechSource, agentPlayout: AgentPlayout, tts: TTS) {
+    this.#speechId = speechId;
+    this.ttsSource = ttsSource;
+    this.#agentPlayout = agentPlayout;
+    this.tts = tts;
+  }
+
+  get speechId(): string {
+    return this.#speechId;
+  }
+
+  get validated(): boolean {
+    return !!this.#playHandle;
+  }
+
+  get interrupted(): boolean {
+    return this.intFut.done;
+  }
+
+  get playHandle(): PlayoutHandle | undefined {
+    return this.#playHandle;
+  }
+
+  /** Validate the speech for playout. */
+  play(): PlayoutHandle {
+    if (this.interrupted) {
+      throw new Error('synthesis was interrupted');
+    }
+
+    this.#playHandle = this.#agentPlayout.play(this.#speechId, this.queue);
+    return this.#playHandle;
+  }
+
+  /** Interrupt the speech. */
+  interrupt() {
+    if (this.interrupted) {
+      return;
+    }
+
+    this.#logger.child({ speechId: this.#speechId }).debug('interrupting synthesis/playout');
+    this.#playHandle?.interrupt();
+    this.intFut.resolve();
+  }
+}
+
+export class AgentOutput {
+  #agentPlayout: AgentPlayout;
+  #tts: TTS;
+  #tasks: CancellablePromise<void>[] = [];
+
+  constructor(agentPlayout: AgentPlayout, tts: TTS) {
+    this.#agentPlayout = agentPlayout;
+    this.#tts = tts;
+  }
+
+  get playout(): AgentPlayout {
+    return this.#agentPlayout;
+  }
+
+  async close() {
+    this.#tasks.forEach((task) => task.cancel());
+    await Promise.all(this.#tasks);
+  }
+
+  synthesize(speechId: string, ttsSource: SpeechSource): SynthesisHandle {
+    const handle = new SynthesisHandle(speechId, ttsSource, this.#agentPlayout, this.#tts);
+    const task = this.#synthesize(handle);
+    this.#tasks.push(task);
+    task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
+    return handle;
+  }
+
+  #synthesize(handle: SynthesisHandle): CancellablePromise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    return new CancellablePromise(async (resolve, _, onCancel) => {
+      const ttsSource = await handle.ttsSource;
+      let task: CancellablePromise<void>;
+      if (typeof ttsSource === 'string') {
+        task = stringSynthesisTask(ttsSource, handle);
+      } else {
+        task = streamSynthesisTask(ttsSource, handle);
+      }
+
+      onCancel(() => {
+        gracefullyCancel(task);
+      });
+
+      try {
+        await Promise.any([task, handle.intFut.await]);
+      } finally {
+        if (handle.intFut.done) {
+          gracefullyCancel(task);
+        }
+      }
+
+      resolve();
+    });
+  }
+}
+
+const stringSynthesisTask = (text: string, handle: SynthesisHandle): CancellablePromise<void> => {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  return new CancellablePromise<void>(async (resolve, _, onCancel) => {
+    let cancelled = false;
+    onCancel(() => {
+      cancelled = true;
+    });
+
+    const ttsStream = handle.tts.stream();
+    ttsStream.pushText(text);
+    ttsStream.flush();
+    ttsStream.endInput();
+    for await (const audio of ttsStream) {
+      if (cancelled || audio === SynthesizeStream.END_OF_STREAM) break;
+      handle.queue.put(audio.frame);
+    }
+    handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
+
+    resolve();
+  });
+};
+
+const streamSynthesisTask = (
+  stream: AsyncIterable<string>,
+  handle: SynthesisHandle,
+): CancellablePromise<void> => {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  return new CancellablePromise<void>(async (resolve, _, onCancel) => {
+    let cancelled = false;
+    onCancel(() => {
+      cancelled = true;
+    });
+
+    const ttsStream = handle.tts.stream();
+    const readGeneratedAudio = async () => {
+      let started = false;
+      for await (const audio of ttsStream) {
+        if (cancelled) break;
+        if (audio === SynthesizeStream.END_OF_STREAM) {
+          if (started) {
+            break;
+          } else {
+            continue;
+          }
+        }
+        handle.queue.put(audio.frame);
+        started = true;
+      }
+      handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
+    };
+    readGeneratedAudio();
+
+    for await (const text of stream) {
+      if (cancelled) break;
+      ttsStream.pushText(text);
+    }
+    ttsStream.flush();
+    ttsStream.endInput();
+
+    resolve();
+  });
+};

package/src/pipeline/agent_playout.ts

@@ -0,0 +1,187 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame, AudioSource } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import EventEmitter from 'node:events';
+import { log } from '../log.js';
+import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+import { SynthesisHandle } from './agent_output.js';
+
+export enum AgentPlayoutEvent {
+  PLAYOUT_STARTED,
+  PLAYOUT_STOPPED,
+}
+
+export type AgentPlayoutCallbacks = {
+  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;
+  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;
+};
+
+export class PlayoutHandle {
+  #speechId: string;
+  #audioSource: AudioSource;
+  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
+  totalPlayedTime?: number;
+  #interrupted = false;
+  pushedDuration = 0;
+  intFut = new Future();
+  doneFut = new Future();
+
+  constructor(
+    speechId: string,
+    audioSource: AudioSource,
+    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+  ) {
+    this.#speechId = speechId;
+    this.#audioSource = audioSource;
+    this.playoutSource = playoutSource;
+  }
+
+  get speechId(): string {
+    return this.#speechId;
+  }
+
+  get interrupted(): boolean {
+    return this.#interrupted;
+  }
+
+  get timePlayed(): number {
+    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;
+  }
+
+  get done(): boolean {
+    return this.doneFut.done || this.#interrupted;
+  }
+
+  interrupt() {
+    if (this.done) {
+      return;
+    }
+
+    this.intFut.resolve();
+    this.#interrupted = true;
+  }
+
+  join(): Future {
+    return this.doneFut;
+  }
+}
+
+export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {
+  #closed = false;
+  #audioSource: AudioSource;
+  #targetVolume = 1;
+  #playoutTask?: CancellablePromise<void>;
+  #logger = log();
+
+  constructor(audioSource: AudioSource) {
+    super();
+    this.#audioSource = audioSource;
+  }
+
+  get targetVolume(): number {
+    return this.#targetVolume;
+  }
+
+  set targetVolume(vol: number) {
+    this.#targetVolume = vol;
+  }
+
+  play(
+    speechId: string,
+    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+  ): PlayoutHandle {
+    if (this.#closed) {
+      throw new Error('source closed');
+    }
+
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+
+    this.#playoutTask = this.#playout(handle, this.#playoutTask);
+    return handle;
+  }
+
+  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {
+    return new CancellablePromise(async (resolve, _, onCancel) => {
+      const cancel = () => {
+        captureTask.cancel();
+        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+
+        if (handle.interrupted || captureTask.error) {
+          this.#audioSource.clearQueue(); // make sure to remove any queued frames
+        }
+
+        if (!firstFrame) {
+          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);
+        }
+
+        handle.doneFut.resolve();
+
+        this.#logger
+          .child({ speechId: handle.speechId, interrupted: handle.interrupted })
+          .debug('playout finished');
+      };
+
+      onCancel(() => {
+        cancel();
+      });
+
+      if (oldTask) {
+        await gracefullyCancel(oldTask);
+      }
+
+      if (this.#audioSource.queuedDuration > 0) {
+        // this should not happen, but log it just in case
+        this.#logger
+          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })
+          .warn('new playout while the source is still playing');
+      }
+
+      let firstFrame = true;
+
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {
+        let cancelled = false;
+        onCancel(() => {
+          cancelled = true;
+        });
+
+        for await (const frame of handle.playoutSource) {
+          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) break;
+          if (firstFrame) {
+            this.#logger
+              .child({ speechId: handle.speechId })
+              .debug('started playing the first time');
+            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);
+            firstFrame = false;
+          }
+          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;
+          await this.#audioSource.captureFrame(frame);
+          await this.#audioSource.waitForPlayout();
+        }
+
+        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,
+        //            but for some reason too many TTS frames can gunk up the buffer and lead to
+        //            FFI errors. this works 🤷‍♀️
+        // if (this.#audioSource.queuedDuration > 0) {
+        //   await this.#audioSource.waitForPlayout();
+        // }
+
+        resolve();
+      });
+
+      try {
+        await Promise.any([captureTask, handle.intFut.await]);
+      } finally {
+        cancel();
+        resolve();
+      }
+    });
+  }
+
+  async close() {
+    this.#closed = true;
+    await this.#playoutTask;
+  }
+}