@livekit/agents 1.1.0 → 1.2.1

package/src/audio.ts

@@ -1,11 +1,28 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import ffmpegInstaller from '@ffmpeg-installer/ffmpeg';
 import { AudioFrame } from '@livekit/rtc-node';
+import ffmpeg from 'fluent-ffmpeg';
+import type { ReadableStream } from 'node:stream/web';
 import { log } from './log.js';
+import { createStreamChannel } from './stream/stream_channel.js';
 import type { AudioBuffer } from './utils.js';
 
-export function calculateAudioDuration(frame: AudioBuffer) {
+ffmpeg.setFfmpegPath(ffmpegInstaller.path);
+
+export interface AudioDecodeOptions {
+  sampleRate?: number;
+  numChannels?: number;
+  /**
+   * Audio format hint (e.g., 'mp3', 'ogg', 'wav', 'opus')
+   * If not provided, FFmpeg will auto-detect
+   */
+  format?: string;
+  abortSignal?: AbortSignal;
+}
+
+export function calculateAudioDurationSeconds(frame: AudioBuffer) {
   // TODO(AJS-102): use frame.durationMs once available in rtc-node
   return Array.isArray(frame)
     ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0)
@@ -72,3 +89,117 @@ export class AudioByteStream {
     return frames;
   }
 }
+
+/**
+ * Decode an audio file into AudioFrame instances
+ *
+ * @param filePath - Path to the audio file
+ * @param options - Decoding options
+ * @returns AsyncGenerator that yields AudioFrame objects
+ *
+ * @example
+ * ```typescript
+ * for await (const frame of audioFramesFromFile('audio.ogg', { sampleRate: 48000 })) {
+ *   console.log('Frame:', frame.samplesPerChannel, 'samples');
+ * }
+ * ```
+ */
+export function audioFramesFromFile(
+  filePath: string,
+  options: AudioDecodeOptions = {},
+): ReadableStream<AudioFrame> {
+  const sampleRate = options.sampleRate ?? 48000;
+  const numChannels = options.numChannels ?? 1;
+
+  const audioStream = new AudioByteStream(sampleRate, numChannels);
+  const channel = createStreamChannel<AudioFrame>();
+  const logger = log();
+
+  // TODO (Brian): decode WAV using a custom decoder instead of FFmpeg
+  const command = ffmpeg(filePath)
+    .inputOptions([
+      '-probesize',
+      '32',
+      '-analyzeduration',
+      '0',
+      '-fflags',
+      '+nobuffer+flush_packets',
+      '-flags',
+      'low_delay',
+    ])
+    .format('s16le') // signed 16-bit little-endian PCM to be consistent cross-platform
+    .audioChannels(numChannels)
+    .audioFrequency(sampleRate);
+
+  let commandRunning = true;
+
+  const onClose = () => {
+    logger.debug('Audio file playback aborted');
+
+    channel.close();
+    if (commandRunning) {
+      commandRunning = false;
+      command.kill('SIGKILL');
+    }
+  };
+
+  const outputStream = command.pipe();
+  options.abortSignal?.addEventListener('abort', onClose, { once: true });
+
+  outputStream.on('data', (chunk: Buffer) => {
+    const arrayBuffer = chunk.buffer.slice(
+      chunk.byteOffset,
+      chunk.byteOffset + chunk.byteLength,
+    ) as ArrayBuffer;
+
+    const frames = audioStream.write(arrayBuffer);
+    for (const frame of frames) {
+      channel.write(frame);
+    }
+  });
+
+  outputStream.on('end', () => {
+    const frames = audioStream.flush();
+    for (const frame of frames) {
+      channel.write(frame);
+    }
+    commandRunning = false;
+    channel.close();
+  });
+
+  outputStream.on('error', (err: Error) => {
+    logger.error(err);
+    commandRunning = false;
+    onClose();
+  });
+
+  return channel.stream();
+}
+
+/**
+ * Loop audio frames from a file indefinitely
+ *
+ * @param filePath - Path to the audio file
+ * @param options - Decoding options
+ * @returns AsyncGenerator that yields AudioFrame objects in an infinite loop
+ */
+export async function* loopAudioFramesFromFile(
+  filePath: string,
+  options: AudioDecodeOptions = {},
+): AsyncGenerator<AudioFrame, void, unknown> {
+  const frames: AudioFrame[] = [];
+  const logger = log();
+
+  for await (const frame of audioFramesFromFile(filePath, options)) {
+    frames.push(frame);
+    yield frame;
+  }
+
+  while (!options.abortSignal?.aborted) {
+    for (const frame of frames) {
+      yield frame;
+    }
+  }
+
+  logger.debug('Audio file playback loop finished');
+}

package/src/beta/index.ts

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export {
+  TaskGroup,
+  type TaskCompletedEvent,
+  type TaskGroupOptions,
+  type TaskGroupResult,
+} from './workflows/index.js';

package/src/beta/workflows/index.ts

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export {
+  TaskGroup,
+  type TaskCompletedEvent,
+  type TaskGroupOptions,
+  type TaskGroupResult,
+} from './task_group.js';

package/src/beta/workflows/task_group.ts

@@ -0,0 +1,203 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { z } from 'zod';
+import type { ChatContext } from '../../llm/chat_context.js';
+import { LLM, ToolError, ToolFlag, tool } from '../../llm/index.js';
+import { AgentTask } from '../../voice/agent.js';
+
+interface FactoryInfo {
+  taskFactory: () => AgentTask;
+  id: string;
+  description: string;
+}
+
+export interface TaskGroupResult {
+  taskResults: Record<string, unknown>;
+}
+
+export interface TaskCompletedEvent {
+  agentTask: AgentTask;
+  taskId: string;
+  result: unknown;
+}
+
+class OutOfScopeError extends ToolError {
+  readonly targetTaskIds: string[];
+
+  constructor(targetTaskIds: string[]) {
+    super('out_of_scope');
+    this.targetTaskIds = targetTaskIds;
+  }
+}
+
+export interface TaskGroupOptions {
+  summarizeChatCtx?: boolean;
+  returnExceptions?: boolean;
+  chatCtx?: ChatContext;
+  onTaskCompleted?: (event: TaskCompletedEvent) => Promise<void>;
+}
+
+export class TaskGroup extends AgentTask<TaskGroupResult> {
+  private _summarizeChatCtx: boolean;
+  private _returnExceptions: boolean;
+  private _visitedTasks = new Set<string>();
+  private _registeredFactories = new Map<string, FactoryInfo>();
+  private _taskCompletedCallback?: (event: TaskCompletedEvent) => Promise<void>;
+  private _currentTask?: AgentTask;
+
+  constructor(options: TaskGroupOptions = {}) {
+    const { summarizeChatCtx = true, returnExceptions = false, chatCtx, onTaskCompleted } = options;
+
+    super({ instructions: '*empty*', chatCtx });
+
+    this._summarizeChatCtx = summarizeChatCtx;
+    this._returnExceptions = returnExceptions;
+    this._taskCompletedCallback = onTaskCompleted;
+  }
+
+  add(task: () => AgentTask, { id, description }: { id: string; description: string }): this {
+    this._registeredFactories.set(id, { taskFactory: task, id, description });
+    return this;
+  }
+
+  async onEnter(): Promise<void> {
+    const taskStack = [...this._registeredFactories.keys()];
+    const taskResults: Record<string, unknown> = {};
+
+    while (taskStack.length > 0) {
+      const taskId = taskStack.shift()!;
+      const factoryInfo = this._registeredFactories.get(taskId)!;
+
+      this._currentTask = factoryInfo.taskFactory();
+
+      const sharedChatCtx = this._chatCtx.copy();
+      await this._currentTask.updateChatCtx(sharedChatCtx);
+
+      const outOfScopeTool = this.buildOutOfScopeTool(taskId);
+      if (outOfScopeTool) {
+        await this._currentTask.updateTools({
+          ...this._currentTask.toolCtx,
+          out_of_scope: outOfScopeTool,
+        });
+      }
+
+      try {
+        this._visitedTasks.add(taskId);
+        const res = await this._currentTask.run();
+
+        // AgentTask handoff merges omit function calls. Re-merge the completed
+        // task context so task-group summarization can incorporate tool results.
+        this._chatCtx.merge(this._currentTask.chatCtx.copy(), {
+          excludeInstructions: true,
+        });
+
+        taskResults[taskId] = res;
+
+        if (this._taskCompletedCallback) {
+          await this._taskCompletedCallback({
+            agentTask: this._currentTask,
+            taskId,
+            result: res,
+          });
+        }
+      } catch (e) {
+        if (e instanceof OutOfScopeError) {
+          taskStack.unshift(taskId);
+          for (let i = e.targetTaskIds.length - 1; i >= 0; i--) {
+            taskStack.unshift(e.targetTaskIds[i]!);
+          }
+          continue;
+        }
+
+        if (this._returnExceptions) {
+          taskResults[taskId] = e;
+          continue;
+        } else {
+          this.complete(e instanceof Error ? e : new Error(String(e)));
+          return;
+        }
+      }
+    }
+
+    try {
+      if (this._summarizeChatCtx) {
+        const sessionLlm = this.session.llm;
+        if (!(sessionLlm instanceof LLM)) {
+          throw new Error('summarizeChatCtx requires a standard LLM on the session');
+        }
+
+        // Keep the full item stream so summarization can distill tool results
+        // into the history summary instead of dropping them up front.
+        const ctxToSummarize = this._chatCtx.copy({
+          excludeInstructions: false,
+          excludeFunctionCall: false,
+          excludeEmptyMessage: false,
+          excludeHandoff: false,
+        });
+
+        const summarizedChatCtx = await ctxToSummarize._summarize(sessionLlm, {
+          keepLastTurns: 0,
+        });
+
+        await this.updateChatCtx(summarizedChatCtx);
+      }
+    } catch (e) {
+      this.complete(new Error(`failed to summarize the chat_ctx: ${e}`));
+      return;
+    }
+
+    this.complete({ taskResults });
+  }
+
+  private buildOutOfScopeTool(activeTaskId: string) {
+    if (this._visitedTasks.size === 0) {
+      return undefined;
+    }
+
+    const regressionTaskIds = new Set(this._visitedTasks);
+    regressionTaskIds.delete(activeTaskId);
+
+    if (regressionTaskIds.size === 0) {
+      return undefined;
+    }
+
+    const taskRepr: Record<string, string> = {};
+    for (const [id, info] of this._registeredFactories) {
+      if (regressionTaskIds.has(id)) {
+        taskRepr[id] = info.description;
+      }
+    }
+
+    const taskIdValues = [...regressionTaskIds] as [string, ...string[]];
+
+    const description =
+      'Call to regress to other tasks according to what the user requested to modify, return the corresponding task ids. ' +
+      'For example, if the user wants to change their email and there is a task with id "email_task" with a description of "Collect the user\'s email", return the id ("get_email_task"). ' +
+      'If the user requests to regress to multiple tasks, such as changing their phone number and email, return both task ids in the order they were requested. ' +
+      `The following are the IDs and their corresponding task description. ${JSON.stringify(taskRepr)}`;
+
+    const currentTask = this._currentTask;
+    const registeredFactories = this._registeredFactories;
+    const visitedTasks = this._visitedTasks;
+
+    return tool({
+      description,
+      flags: ToolFlag.IGNORE_ON_ENTER,
+      parameters: z.object({
+        task_ids: z.array(z.enum(taskIdValues)).describe('The IDs of the tasks requested'),
+      }),
+      execute: async ({ task_ids }: { task_ids: string[] }) => {
+        for (const tid of task_ids) {
+          if (!registeredFactories.has(tid) || !visitedTasks.has(tid)) {
+            throw new ToolError(`Unable to regress, invalid task id ${tid}`);
+          }
+        }
+
+        if (currentTask && !currentTask.done) {
+          currentTask.complete(new OutOfScopeError(task_ids));
+        }
+      },
+    });
+  }
+}

package/src/cli.ts

@@ -6,10 +6,10 @@ import type { EventEmitter } from 'node:events';
 import { initializeLogger, log } from './log.js';
 import { Plugin } from './plugin.js';
 import { version } from './version.js';
-import { Worker, WorkerOptions } from './worker.js';
+import { AgentServer, ServerOptions } from './worker.js';
 
 type CliArgs = {
-  opts: WorkerOptions;
+  opts: ServerOptions;
   production: boolean;
   watch: boolean;
   event?: EventEmitter;
@@ -17,18 +17,18 @@ type CliArgs = {
   participantIdentity?: string;
 };
 
-const runWorker = async (args: CliArgs) => {
+const runServer = async (args: CliArgs) => {
   initializeLogger({ pretty: !args.production, level: args.opts.logLevel });
   const logger = log();
 
-  // though `production` is defined in WorkerOptions, it will always be overridden by CLI.
+  // though `production` is defined in ServerOptions, it will always be overridden by CLI.
   const { production: _, ...opts } = args.opts; // eslint-disable-line @typescript-eslint/no-unused-vars
-  const worker = new Worker(new WorkerOptions({ production: args.production, ...opts }));
+  const server = new AgentServer(new ServerOptions({ production: args.production, ...opts }));
 
   if (args.room) {
-    worker.event.once('worker_registered', () => {
+    server.event.once('worker_registered', () => {
       logger.info(`connecting to room ${args.room}`);
-      worker.simulateJob(args.room!, args.participantIdentity);
+      server.simulateJob(args.room!, args.participantIdentity);
     });
   }
 
@@ -40,9 +40,9 @@ const runWorker = async (args: CliArgs) => {
       process.exit(130); // SIGINT exit code
     });
     if (args.production) {
-      await worker.drain();
+      await server.drain();
     }
-    await worker.close();
+    await server.close();
     logger.debug('worker closed due to SIGINT.');
     process.exit(130); // SIGINT exit code
   });
@@ -50,15 +50,15 @@ const runWorker = async (args: CliArgs) => {
   process.once('SIGTERM', async () => {
     logger.debug('SIGTERM received in CLI.');
     if (args.production) {
-      await worker.drain();
+      await server.drain();
     }
-    await worker.close();
+    await server.close();
     logger.debug('worker closed due to SIGTERM.');
     process.exit(143); // SIGTERM exit code
   });
 
   try {
-    await worker.run();
+    await server.run();
   } catch {
     logger.fatal('closing worker due to error.');
     process.exit(1);
@@ -72,21 +72,21 @@ const runWorker = async (args: CliArgs) => {
  * @example
  * ```
  * if (process.argv[1] === fileURLToPath(import.meta.url)) {
- *   cli.runApp(new WorkerOptions({ agent: import.meta.filename }));
+ *   cli.runApp(new ServerOptions({ agent: import.meta.filename }));
  * }
  * ```
  */
-export const runApp = (opts: WorkerOptions) => {
+export const runApp = (opts: ServerOptions) => {
+  const logLevelOption = (defaultLevel: string) =>
+    new Option('--log-level <level>', 'Set the logging level')
+      .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
+      .default(defaultLevel)
+      .env('LOG_LEVEL');
+
   const program = new Command()
     .name('agents')
     .description('LiveKit Agents CLI')
     .version(version)
-    .addOption(
-      new Option('--log-level <level>', 'Set the logging level')
-        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
-        .default('info')
-        .env('LOG_LEVEL'),
-    )
     .addOption(
       new Option('--url <string>', 'LiveKit server or Cloud project websocket URL').env(
         'LIVEKIT_URL',
@@ -120,14 +120,16 @@ export const runApp = (opts: WorkerOptions) => {
   program
     .command('start')
     .description('Start the worker in production mode')
-    .action(() => {
-      const options = program.optsWithGlobals();
-      opts.wsURL = options.url || opts.wsURL;
-      opts.apiKey = options.apiKey || opts.apiKey;
-      opts.apiSecret = options.apiSecret || opts.apiSecret;
-      opts.logLevel = options.logLevel || opts.logLevel;
-      opts.workerToken = options.workerToken || opts.workerToken;
-      runWorker({
+    .addOption(logLevelOption('info'))
+    .action((...[, command]) => {
+      const globalOptions = program.optsWithGlobals();
+      const commandOptions = command.opts();
+      opts.wsURL = globalOptions.url || opts.wsURL;
+      opts.apiKey = globalOptions.apiKey || opts.apiKey;
+      opts.apiSecret = globalOptions.apiSecret || opts.apiSecret;
+      opts.logLevel = commandOptions.logLevel;
+      opts.workerToken = globalOptions.workerToken || opts.workerToken;
+      runServer({
         opts,
         production: true,
         watch: false,
@@ -137,20 +139,17 @@ export const runApp = (opts: WorkerOptions) => {
   program
     .command('dev')
     .description('Start the worker in development mode')
-    .addOption(
-      new Option('--log-level <level>', 'Set the logging level')
-        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
-        .default('debug')
-        .env('LOG_LEVEL'),
-    )
-    .action(() => {
-      const options = program.optsWithGlobals();
-      opts.wsURL = options.url || opts.wsURL;
-      opts.apiKey = options.apiKey || opts.apiKey;
-      opts.apiSecret = options.apiSecret || opts.apiSecret;
-      opts.logLevel = options.logLevel || opts.logLevel;
-      opts.workerToken = options.workerToken || opts.workerToken;
-      runWorker({
+    .addOption(logLevelOption('debug'))
+    .action((...[, command]) => {
+      const globalOptions = program.optsWithGlobals();
+      const commandOptions = command.opts();
+      opts.wsURL = globalOptions.url || opts.wsURL;
+      opts.apiKey = globalOptions.apiKey || opts.apiKey;
+      opts.apiSecret = globalOptions.apiSecret || opts.apiSecret;
+      opts.logLevel = commandOptions.logLevel;
+      opts.workerToken = globalOptions.workerToken || opts.workerToken;
+      process.env.LIVEKIT_DEV_MODE = '1';
+      runServer({
         opts,
         production: false,
         watch: false,
@@ -162,40 +161,32 @@ export const runApp = (opts: WorkerOptions) => {
     .description('Connect to a specific room')
     .requiredOption('--room <string>', 'Room name to connect to')
     .option('--participant-identity <string>', 'Identity of user to listen to')
-    .addOption(
-      new Option('--log-level <level>', 'Set the logging level')
-        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
-        .default('debug')
-        .env('LOG_LEVEL'),
-    )
+    .addOption(logLevelOption('info'))
     .action((...[, command]) => {
-      const options = command.optsWithGlobals();
-      opts.wsURL = options.url || opts.wsURL;
-      opts.apiKey = options.apiKey || opts.apiKey;
-      opts.apiSecret = options.apiSecret || opts.apiSecret;
-      opts.logLevel = options.logLevel || opts.logLevel;
-      opts.workerToken = options.workerToken || opts.workerToken;
-      runWorker({
+      const globalOptions = program.optsWithGlobals();
+      const commandOptions = command.opts();
+      opts.wsURL = globalOptions.url || opts.wsURL;
+      opts.apiKey = globalOptions.apiKey || opts.apiKey;
+      opts.apiSecret = globalOptions.apiSecret || opts.apiSecret;
+      opts.logLevel = commandOptions.logLevel;
+      opts.workerToken = globalOptions.workerToken || opts.workerToken;
+      process.env.LIVEKIT_DEV_MODE = '1';
+      runServer({
         opts,
         production: false,
         watch: false,
-        room: options.room,
-        participantIdentity: options.participantIdentity,
+        room: commandOptions.room,
+        participantIdentity: commandOptions.participantIdentity,
       });
     });
 
   program
     .command('download-files')
     .description('Download plugin dependency files')
-    .addOption(
-      new Option('--log-level <level>', 'Set the logging level')
-        .choices(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
-        .default('debug')
-        .env('LOG_LEVEL'),
-    )
-    .action(() => {
-      const options = program.optsWithGlobals();
-      initializeLogger({ pretty: true, level: options.logLevel });
+    .addOption(logLevelOption('debug'))
+    .action((...[, command]) => {
+      const commandOptions = command.opts();
+      initializeLogger({ pretty: true, level: commandOptions.logLevel });
       const logger = log();
 
       const downloadFiles = async () => {