@livekit/agents 0.3.5 → 0.4.1

package/src/llm/function_context.ts

@@ -18,6 +18,24 @@ export interface CallableFunction<P extends z.ZodTypeAny = any, R = any> {
   execute: (args: inferParameters<P>) => PromiseLike<R>;
 }
 
+/** A function that has been called but is not yet running */
+export interface FunctionCallInfo<P extends z.ZodTypeAny = any, R = any> {
+  name: string;
+  func: CallableFunction<P, R>;
+  toolCallId: string;
+  rawParams: string;
+  params: inferParameters<P>;
+  task?: PromiseLike<CallableFunctionResult>;
+}
+
+/** The result of a ran FunctionCallInfo. */
+export interface CallableFunctionResult {
+  name: string;
+  toolCallId: string;
+  result?: any;
+  error?: any;
+}
+
 /** An object containing callable functions and their names */
 export type FunctionContext = {
   [name: string]: CallableFunction;
@@ -26,29 +44,50 @@ export type FunctionContext = {
 /** @internal */
 export const oaiParams = (p: z.AnyZodObject) => {
   const properties: Record<string, any> = {};
-  const required_properties: string[] = [];
+  const requiredProperties: string[] = [];
 
-  for (const key in p.shape) {
-    const field = p.shape[key];
-    const description = field._def.description || undefined;
-    let type: string;
-    let enumValues: any[] | undefined;
+  const processZodType = (field: z.ZodTypeAny): any => {
+    const isOptional = field instanceof z.ZodOptional;
+    const nestedField = isOptional ? field._def.innerType : field;
+    const description = field._def.description;
 
-    if (field instanceof z.ZodEnum) {
-      enumValues = field._def.values;
-      type = typeof enumValues![0];
+    if (nestedField instanceof z.ZodEnum) {
+      return {
+        type: typeof nestedField._def.values[0],
+        ...(description && { description }),
+        enum: nestedField._def.values,
+      };
+    } else if (nestedField instanceof z.ZodArray) {
+      const elementType = nestedField._def.type;
+      return {
+        type: 'array',
+        ...(description && { description }),
+        items: processZodType(elementType),
+      };
+    } else if (nestedField instanceof z.ZodObject) {
+      const { properties, required } = oaiParams(nestedField);
+      return {
+        type: 'object',
+        ...(description && { description }),
+        properties,
+        required,
+      };
     } else {
-      type = field._def.typeName.toLowerCase();
+      let type = nestedField._def.typeName.toLowerCase();
+      type = type.includes('zod') ? type.substring(3) : type;
+      return {
+        type,
+        ...(description && { description }),
+      };
     }
+  };
 
-    properties[key] = {
-      type: type.includes('zod') ? type.substring(3) : type,
-      description,
-      enum: enumValues,
-    };
+  for (const key in p.shape) {
+    const field = p.shape[key];
+    properties[key] = processZodType(field);
 
-    if (!field._def.defaultValue) {
-      required_properties.push(key);
+    if (!(field instanceof z.ZodOptional)) {
+      requiredProperties.push(key);
     }
   }
 
@@ -56,6 +95,26 @@ export const oaiParams = (p: z.AnyZodObject) => {
   return {
     type,
     properties,
-    required_properties,
+    required: requiredProperties,
+  };
+};
+
+/** @internal */
+export const oaiBuildFunctionInfo = (
+  fncCtx: FunctionContext,
+  toolCallId: string,
+  fncName: string,
+  rawArgs: string,
+): FunctionCallInfo => {
+  if (!fncCtx[fncName]) {
+    throw new Error(`AI function ${fncName} not found`);
+  }
+
+  return {
+    name: fncName,
+    func: fncCtx[fncName],
+    toolCallId,
+    rawParams: rawArgs,
+    params: JSON.parse(rawArgs),
   };
 };

package/src/llm/index.ts

@@ -1,11 +1,30 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import {
+export {
   type CallableFunction,
+  type FunctionCallInfo,
+  type CallableFunctionResult,
   type FunctionContext,
   type inferParameters,
   oaiParams,
+  oaiBuildFunctionInfo,
 } from './function_context.js';
 
-export { CallableFunction, FunctionContext, inferParameters, oaiParams };
+export {
+  type ChatImage,
+  type ChatAudio,
+  type ChatContent,
+  ChatRole,
+  ChatMessage,
+  ChatContext,
+} from './chat_context.js';
+
+export {
+  type ChoiceDelta,
+  type CompletionUsage,
+  type Choice,
+  type ChatChunk,
+  LLM,
+  LLMStream,
+} from './llm.js';

package/src/llm/llm.ts

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { AsyncIterableQueue } from '../utils.js';
+import type { ChatContext, ChatRole } from './chat_context.js';
+import type { FunctionCallInfo, FunctionContext } from './function_context.js';
+
+export interface ChoiceDelta {
+  role: ChatRole;
+  content?: string;
+  toolCalls?: FunctionCallInfo[];
+}
+
+export interface CompletionUsage {
+  completionTokens: number;
+  promptTokens: number;
+  totalTokens: number;
+}
+
+export interface Choice {
+  delta: ChoiceDelta;
+  index: number;
+}
+
+export interface ChatChunk {
+  requestId: string;
+  choices: Choice[];
+  usage?: CompletionUsage;
+}
+
+export abstract class LLM {
+  /**
+   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.
+   */
+  abstract chat({
+    chatCtx,
+    fncCtx,
+    temperature,
+    n,
+    parallelToolCalls,
+  }: {
+    chatCtx: ChatContext;
+    fncCtx?: FunctionContext;
+    temperature?: number;
+    n?: number;
+    parallelToolCalls?: boolean;
+  }): LLMStream;
+}
+
+export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
+  protected queue = new AsyncIterableQueue<ChatChunk>();
+  protected closed = false;
+  protected _functionCalls: FunctionCallInfo[] = [];
+
+  #chatCtx: ChatContext;
+  #fncCtx?: FunctionContext;
+
+  constructor(chatCtx: ChatContext, fncCtx?: FunctionContext) {
+    this.#chatCtx = chatCtx;
+    this.#fncCtx = fncCtx;
+  }
+
+  /** List of called functions from this stream. */
+  get functionCalls(): FunctionCallInfo[] {
+    return this._functionCalls;
+  }
+
+  /** The function context of this stream. */
+  get fncCtx(): FunctionContext | undefined {
+    return this.#fncCtx;
+  }
+
+  /** The initial chat context of this stream. */
+  get chatCtx(): ChatContext {
+    return this.#chatCtx;
+  }
+
+  /** Execute all deferred functions of this stream concurrently. */
+  executeFunctions(): FunctionCallInfo[] {
+    this._functionCalls.forEach(
+      (f) =>
+        (f.task = f.func.execute(f.params).then(
+          (result) => ({ name: f.name, toolCallId: f.toolCallId, result }),
+          (error) => ({ name: f.name, toolCallId: f.toolCallId, error }),
+        )),
+    );
+    return this._functionCalls;
+  }
+
+  next(): Promise<IteratorResult<ChatChunk>> {
+    return this.queue.next();
+  }
+
+  close() {
+    this.queue.close();
+    this.closed = true;
+  }
+
+  [Symbol.asyncIterator](): LLMStream {
+    return this;
+  }
+}

package/src/multimodal/multimodal_agent.ts

@@ -64,13 +64,16 @@ export class MultimodalAgent extends EventEmitter {
 
   constructor({
     model,
+    chatCtx,
     fncCtx,
   }: {
     model: RealtimeModel;
-    fncCtx?: llm.FunctionContext | undefined;
+    chatCtx?: llm.ChatContext;
+    fncCtx?: llm.FunctionContext;
   }) {
     super();
     this.model = model;
+    this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
   }
 
@@ -83,6 +86,7 @@ export class MultimodalAgent extends EventEmitter {
   #logger = log();
   #session: RealtimeSession | null = null;
   #fncCtx: llm.FunctionContext | undefined = undefined;
+  #chatCtx: llm.ChatContext | undefined = undefined;
 
   #_started: boolean = false;
   #_pendingFunctionCalls: Set<string> = new Set();
@@ -209,7 +213,7 @@ export class MultimodalAgent extends EventEmitter {
         }
       }
 
-      this.#session = this.model.session({ fncCtx: this.#fncCtx });
+      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });
       this.#started = true;
 
       // eslint-disable-next-line @typescript-eslint/no-explicit-any

package/src/pipeline/agent_output.ts

@@ -0,0 +1,185 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import { log } from '../log.js';
+import { SynthesizeStream, type TTS } from '../tts/index.js';
+import { AsyncIterableQueue, CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+import type { AgentPlayout, PlayoutHandle } from './agent_playout.js';
+
+export type SpeechSource = AsyncIterable<string> | string | Promise<string>;
+
+export class SynthesisHandle {
+  static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+
+  #speechId: string;
+  ttsSource: SpeechSource;
+  #agentPlayout: AgentPlayout;
+  tts: TTS;
+  queue = new AsyncIterableQueue<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>();
+  #playHandle?: PlayoutHandle;
+  intFut = new Future();
+  #logger = log();
+
+  constructor(speechId: string, ttsSource: SpeechSource, agentPlayout: AgentPlayout, tts: TTS) {
+    this.#speechId = speechId;
+    this.ttsSource = ttsSource;
+    this.#agentPlayout = agentPlayout;
+    this.tts = tts;
+  }
+
+  get speechId(): string {
+    return this.#speechId;
+  }
+
+  get validated(): boolean {
+    return !!this.#playHandle;
+  }
+
+  get interrupted(): boolean {
+    return this.intFut.done;
+  }
+
+  get playHandle(): PlayoutHandle | undefined {
+    return this.#playHandle;
+  }
+
+  /** Validate the speech for playout. */
+  play(): PlayoutHandle {
+    if (this.interrupted) {
+      throw new Error('synthesis was interrupted');
+    }
+
+    this.#playHandle = this.#agentPlayout.play(this.#speechId, this.queue);
+    return this.#playHandle;
+  }
+
+  /** Interrupt the speech. */
+  interrupt() {
+    if (this.interrupted) {
+      return;
+    }
+
+    this.#logger.child({ speechId: this.#speechId }).debug('interrupting synthesis/playout');
+    this.#playHandle?.interrupt();
+    this.intFut.resolve();
+  }
+}
+
+export class AgentOutput {
+  #agentPlayout: AgentPlayout;
+  #tts: TTS;
+  #tasks: CancellablePromise<void>[] = [];
+
+  constructor(agentPlayout: AgentPlayout, tts: TTS) {
+    this.#agentPlayout = agentPlayout;
+    this.#tts = tts;
+  }
+
+  get playout(): AgentPlayout {
+    return this.#agentPlayout;
+  }
+
+  async close() {
+    this.#tasks.forEach((task) => task.cancel());
+    await Promise.all(this.#tasks);
+  }
+
+  synthesize(speechId: string, ttsSource: SpeechSource): SynthesisHandle {
+    const handle = new SynthesisHandle(speechId, ttsSource, this.#agentPlayout, this.#tts);
+    const task = this.#synthesize(handle);
+    this.#tasks.push(task);
+    task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
+    return handle;
+  }
+
+  #synthesize(handle: SynthesisHandle): CancellablePromise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    return new CancellablePromise(async (resolve, _, onCancel) => {
+      const ttsSource = await handle.ttsSource;
+      let task: CancellablePromise<void>;
+      if (typeof ttsSource === 'string') {
+        task = stringSynthesisTask(ttsSource, handle);
+      } else {
+        task = streamSynthesisTask(ttsSource, handle);
+      }
+
+      onCancel(() => {
+        gracefullyCancel(task);
+      });
+
+      try {
+        await Promise.any([task, handle.intFut.await]);
+      } finally {
+        if (handle.intFut.done) {
+          gracefullyCancel(task);
+        }
+      }
+
+      resolve();
+    });
+  }
+}
+
+const stringSynthesisTask = (text: string, handle: SynthesisHandle): CancellablePromise<void> => {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  return new CancellablePromise<void>(async (resolve, _, onCancel) => {
+    let cancelled = false;
+    onCancel(() => {
+      cancelled = true;
+    });
+
+    const ttsStream = handle.tts.stream();
+    ttsStream.pushText(text);
+    ttsStream.flush();
+    ttsStream.endInput();
+    for await (const audio of ttsStream) {
+      if (cancelled || audio === SynthesizeStream.END_OF_STREAM) break;
+      handle.queue.put(audio.frame);
+    }
+    handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
+
+    resolve();
+  });
+};
+
+const streamSynthesisTask = (
+  stream: AsyncIterable<string>,
+  handle: SynthesisHandle,
+): CancellablePromise<void> => {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  return new CancellablePromise<void>(async (resolve, _, onCancel) => {
+    let cancelled = false;
+    onCancel(() => {
+      cancelled = true;
+    });
+
+    const ttsStream = handle.tts.stream();
+    const readGeneratedAudio = async () => {
+      let started = false;
+      for await (const audio of ttsStream) {
+        if (cancelled) break;
+        if (audio === SynthesizeStream.END_OF_STREAM) {
+          if (started) {
+            break;
+          } else {
+            continue;
+          }
+        }
+        handle.queue.put(audio.frame);
+        started = true;
+      }
+      handle.queue.put(SynthesisHandle.FLUSH_SENTINEL);
+    };
+    readGeneratedAudio();
+
+    for await (const text of stream) {
+      if (cancelled) break;
+      ttsStream.pushText(text);
+    }
+    ttsStream.flush();
+    ttsStream.endInput();
+
+    resolve();
+  });
+};

package/src/pipeline/agent_playout.ts

@@ -0,0 +1,187 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame, AudioSource } from '@livekit/rtc-node';
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import EventEmitter from 'node:events';
+import { log } from '../log.js';
+import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+import { SynthesisHandle } from './agent_output.js';
+
+export enum AgentPlayoutEvent {
+  PLAYOUT_STARTED,
+  PLAYOUT_STOPPED,
+}
+
+export type AgentPlayoutCallbacks = {
+  [AgentPlayoutEvent.PLAYOUT_STARTED]: () => void;
+  [AgentPlayoutEvent.PLAYOUT_STOPPED]: (interrupt: boolean) => void;
+};
+
+export class PlayoutHandle {
+  #speechId: string;
+  #audioSource: AudioSource;
+  playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>;
+  totalPlayedTime?: number;
+  #interrupted = false;
+  pushedDuration = 0;
+  intFut = new Future();
+  doneFut = new Future();
+
+  constructor(
+    speechId: string,
+    audioSource: AudioSource,
+    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+  ) {
+    this.#speechId = speechId;
+    this.#audioSource = audioSource;
+    this.playoutSource = playoutSource;
+  }
+
+  get speechId(): string {
+    return this.#speechId;
+  }
+
+  get interrupted(): boolean {
+    return this.#interrupted;
+  }
+
+  get timePlayed(): number {
+    return this.totalPlayedTime || this.pushedDuration - this.#audioSource.queuedDuration;
+  }
+
+  get done(): boolean {
+    return this.doneFut.done || this.#interrupted;
+  }
+
+  interrupt() {
+    if (this.done) {
+      return;
+    }
+
+    this.intFut.resolve();
+    this.#interrupted = true;
+  }
+
+  join(): Future {
+    return this.doneFut;
+  }
+}
+
+export class AgentPlayout extends (EventEmitter as new () => TypedEmitter<AgentPlayoutCallbacks>) {
+  #closed = false;
+  #audioSource: AudioSource;
+  #targetVolume = 1;
+  #playoutTask?: CancellablePromise<void>;
+  #logger = log();
+
+  constructor(audioSource: AudioSource) {
+    super();
+    this.#audioSource = audioSource;
+  }
+
+  get targetVolume(): number {
+    return this.#targetVolume;
+  }
+
+  set targetVolume(vol: number) {
+    this.#targetVolume = vol;
+  }
+
+  play(
+    speechId: string,
+    playoutSource: AsyncIterable<AudioFrame | typeof SynthesisHandle.FLUSH_SENTINEL>,
+  ): PlayoutHandle {
+    if (this.#closed) {
+      throw new Error('source closed');
+    }
+
+    const handle = new PlayoutHandle(speechId, this.#audioSource, playoutSource);
+
+    this.#playoutTask = this.#playout(handle, this.#playoutTask);
+    return handle;
+  }
+
+  #playout(handle: PlayoutHandle, oldTask?: CancellablePromise<void>): CancellablePromise<void> {
+    return new CancellablePromise(async (resolve, _, onCancel) => {
+      const cancel = () => {
+        captureTask.cancel();
+        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+
+        if (handle.interrupted || captureTask.error) {
+          this.#audioSource.clearQueue(); // make sure to remove any queued frames
+        }
+
+        if (!firstFrame) {
+          this.emit(AgentPlayoutEvent.PLAYOUT_STOPPED, handle.interrupted);
+        }
+
+        handle.doneFut.resolve();
+
+        this.#logger
+          .child({ speechId: handle.speechId, interrupted: handle.interrupted })
+          .debug('playout finished');
+      };
+
+      onCancel(() => {
+        cancel();
+      });
+
+      if (oldTask) {
+        await gracefullyCancel(oldTask);
+      }
+
+      if (this.#audioSource.queuedDuration > 0) {
+        // this should not happen, but log it just in case
+        this.#logger
+          .child({ speechId: handle.speechId, queuedDuration: this.#audioSource.queuedDuration })
+          .warn('new playout while the source is still playing');
+      }
+
+      let firstFrame = true;
+
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const captureTask = new CancellablePromise<void>(async (resolve, _, onCancel) => {
+        let cancelled = false;
+        onCancel(() => {
+          cancelled = true;
+        });
+
+        for await (const frame of handle.playoutSource) {
+          if (cancelled || frame === SynthesisHandle.FLUSH_SENTINEL) break;
+          if (firstFrame) {
+            this.#logger
+              .child({ speechId: handle.speechId })
+              .debug('started playing the first time');
+            this.emit(AgentPlayoutEvent.PLAYOUT_STARTED);
+            firstFrame = false;
+          }
+          handle.pushedDuration += (frame.samplesPerChannel / frame.sampleRate) * 1000;
+          await this.#audioSource.captureFrame(frame);
+          await this.#audioSource.waitForPlayout();
+        }
+
+        // XXX(nbsp): line 161 waits instead of this. this is not the case on python agents,
+        //            but for some reason too many TTS frames can gunk up the buffer and lead to
+        //            FFI errors. this works 🤷‍♀️
+        // if (this.#audioSource.queuedDuration > 0) {
+        //   await this.#audioSource.waitForPlayout();
+        // }
+
+        resolve();
+      });
+
+      try {
+        await Promise.any([captureTask, handle.intFut.await]);
+      } finally {
+        cancel();
+        resolve();
+      }
+    });
+  }
+
+  async close() {
+    this.#closed = true;
+    await this.#playoutTask;
+  }
+}