@livekit/agents 1.1.0 → 1.2.0

package/src/voice/generation_tools.test.ts

@@ -0,0 +1,268 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { ReadableStream as NodeReadableStream } from 'stream/web';
+import { describe, expect, it } from 'vitest';
+import { z } from 'zod';
+import { FunctionCall, tool } from '../llm/index.js';
+import { initializeLogger } from '../log.js';
+import type { Task } from '../utils.js';
+import { cancelAndWait, delay } from '../utils.js';
+import { type _TextOut, performTextForwarding, performToolExecutions } from './generation.js';
+
+function createStringStream(chunks: string[], delayMs: number = 0): NodeReadableStream<string> {
+  return new NodeReadableStream<string>({
+    async start(controller) {
+      for (const c of chunks) {
+        if (delayMs > 0) {
+          await delay(delayMs);
+        }
+        controller.enqueue(c);
+      }
+      controller.close();
+    },
+  });
+}
+
+function createFunctionCallStream(fc: FunctionCall): NodeReadableStream<FunctionCall> {
+  return new NodeReadableStream<FunctionCall>({
+    start(controller) {
+      controller.enqueue(fc);
+      controller.close();
+    },
+  });
+}
+
+function createFunctionCallStreamFromArray(fcs: FunctionCall[]): NodeReadableStream<FunctionCall> {
+  return new NodeReadableStream<FunctionCall>({
+    start(controller) {
+      for (const fc of fcs) {
+        controller.enqueue(fc);
+      }
+      controller.close();
+    },
+  });
+}
+
+describe('Generation + Tool Execution', () => {
+  initializeLogger({ pretty: false, level: 'silent' });
+
+  it('should not abort tool when preamble forwarders are cleaned up', async () => {
+    const replyAbortController = new AbortController();
+    const forwarderController = new AbortController();
+
+    const chunks = Array.from({ length: 50 }, () => `Hi.`);
+    const fullPreambleText = chunks.join('');
+    const preamble = createStringStream(chunks, 20);
+    const [textForwardTask, textOut]: [Task<void>, _TextOut] = performTextForwarding(
+      preamble,
+      forwarderController,
+      null,
+    );
+
+    // Tool that takes > 5 seconds
+    let toolAborted = false;
+    const getWeather = tool({
+      description: 'weather',
+      parameters: z.object({ location: z.string() }),
+      execute: async ({ location }, { abortSignal }) => {
+        if (abortSignal) {
+          abortSignal.addEventListener('abort', () => {
+            toolAborted = true;
+          });
+        }
+        // 6s delay
+        await delay(6000);
+        return `Sunny in ${location}`;
+      },
+    });
+
+    const fc = FunctionCall.create({
+      callId: 'call_1',
+      name: 'getWeather',
+      args: JSON.stringify({ location: 'San Francisco' }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_test', _itemAdded: () => {} } as any,
+      toolCtx: { getWeather } as any,
+      toolCallStream,
+      controller: replyAbortController,
+      onToolExecutionStarted: () => {},
+      onToolExecutionCompleted: () => {},
+    });
+
+    // Ensure tool has started, then cancel forwarders mid-stream (without aborting parent AbortController)
+    await toolOutput.firstToolStartedFuture.await;
+    await delay(100);
+    await cancelAndWait([textForwardTask], 5000);
+
+    await execTask.result;
+
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0]!;
+    expect(out.toolCallOutput?.isError).toBe(false);
+    expect(out.toolCallOutput?.output).toContain('Sunny in San Francisco');
+    // Forwarder should have been cancelled before finishing all preamble chunks
+    expect(textOut.text).not.toBe(fullPreambleText);
+    // Tool's abort signal must not have fired
+    expect(toolAborted).toBe(false);
+  }, 30_000);
+
+  it('should return basic tool execution output', async () => {
+    const replyAbortController = new AbortController();
+
+    const echo = tool({
+      description: 'echo',
+      parameters: z.object({ msg: z.string() }),
+      execute: async ({ msg }) => `echo: ${msg}`,
+    });
+
+    const fc = FunctionCall.create({
+      callId: 'call_2',
+      name: 'echo',
+      args: JSON.stringify({ msg: 'hello' }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_test2', _itemAdded: () => {} } as any,
+      toolCtx: { echo } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await execTask.result;
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0];
+    expect(out?.toolCallOutput?.isError).toBe(false);
+    expect(out?.toolCallOutput?.output).toContain('echo: hello');
+  });
+
+  it('should abort tool when reply is aborted mid-execution', async () => {
+    const replyAbortController = new AbortController();
+
+    let aborted = false;
+    const longOp = tool({
+      description: 'longOp',
+      parameters: z.object({ ms: z.number() }),
+      execute: async ({ ms }, { abortSignal }) => {
+        if (abortSignal) {
+          abortSignal.addEventListener('abort', () => {
+            aborted = true;
+          });
+        }
+        await delay(ms);
+        return 'done';
+      },
+    });
+
+    const fc = FunctionCall.create({
+      callId: 'call_abort_1',
+      name: 'longOp',
+      args: JSON.stringify({ ms: 5000 }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_abort', _itemAdded: () => {} } as any,
+      toolCtx: { longOp } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await toolOutput.firstToolStartedFuture.await;
+    replyAbortController.abort();
+    await execTask.result;
+
+    expect(aborted).toBe(true);
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0];
+    expect(out?.toolCallOutput?.isError).toBe(true);
+  }, 20_000);
+
+  it('should return error output on invalid tool args (zod validation failure)', async () => {
+    const replyAbortController = new AbortController();
+
+    const echo = tool({
+      description: 'echo',
+      parameters: z.object({ msg: z.string() }),
+      execute: async ({ msg }) => `echo: ${msg}`,
+    });
+
+    // invalid: msg should be string
+    const fc = FunctionCall.create({
+      callId: 'call_invalid_args',
+      name: 'echo',
+      args: JSON.stringify({ msg: 123 }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_invalid', _itemAdded: () => {} } as any,
+      toolCtx: { echo } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await execTask.result;
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0];
+    expect(out?.toolCallOutput?.isError).toBe(true);
+  });
+
+  it('should handle multiple tool calls within a single stream', async () => {
+    const replyAbortController = new AbortController();
+
+    const sum = tool({
+      description: 'sum',
+      parameters: z.object({ a: z.number(), b: z.number() }),
+      execute: async ({ a, b }) => a + b,
+    });
+    const upper = tool({
+      description: 'upper',
+      parameters: z.object({ s: z.string() }),
+      execute: async ({ s }) => s.toUpperCase(),
+    });
+
+    const fc1 = FunctionCall.create({
+      callId: 'call_multi_1',
+      name: 'sum',
+      args: JSON.stringify({ a: 2, b: 3 }),
+    });
+    const fc2 = FunctionCall.create({
+      callId: 'call_multi_2',
+      name: 'upper',
+      args: JSON.stringify({ s: 'hey' }),
+    });
+    const toolCallStream = createFunctionCallStreamFromArray([fc1, fc2]);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_multi', _itemAdded: () => {} } as any,
+      toolCtx: { sum, upper } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await execTask.result;
+    expect(toolOutput.output.length).toBe(2);
+
+    // sort by callId to assert deterministically
+    const sorted = [...toolOutput.output].sort((a, b) =>
+      a.toolCall.callId.localeCompare(b.toolCall.callId),
+    );
+
+    expect(sorted[0]?.toolCall.name).toBe('sum');
+    expect(sorted[0]?.toolCallOutput?.isError).toBe(false);
+    expect(sorted[0]?.toolCallOutput?.output).toBe('5');
+    expect(sorted[1]?.toolCall.name).toBe('upper');
+    expect(sorted[1]?.toolCallOutput?.isError).toBe(false);
+    expect(sorted[1]?.toolCallOutput?.output).toBe('"HEY"');
+  });
+});

package/src/voice/index.ts

@@ -1,9 +1,23 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-export { Agent, StopResponse, type AgentOptions, type ModelSettings } from './agent.js';
-export { AgentSession, type AgentSessionOptions } from './agent_session.js';
-
+export { Agent, AgentTask, StopResponse, type AgentOptions, type ModelSettings } from './agent.js';
+export { AgentSession, type AgentSessionOptions, type VoiceOptions } from './agent_session.js';
 export * from './avatar/index.js';
+export * from './background_audio.js';
+export {
+  type TextInputCallback,
+  type TextInputEvent,
+  RemoteSession,
+  type RemoteSessionCallbacks,
+  type RemoteSessionEventTypes,
+  SessionHost,
+  SessionTransport,
+  RoomSessionTransport,
+} from './remote_session.js';
 export * from './events.js';
+export { type TimedString } from './io.js';
+export * from './report.js';
+export * from './room_io/index.js';
 export { RunContext } from './run_context.js';
+export * as testing from './testing/index.js';

package/src/voice/interruption_detection.test.ts

@@ -0,0 +1,151 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Unit tests for interruption detection logic in AgentActivity.
+ *
+ * Tests the refactored minInterruptionWords check which ensures:
+ * - Consistent word count filtering across all speech scenarios
+ * - Proper handling of empty strings, undefined, and short speech
+ * - Interruptions allowed only when word count meets or exceeds minInterruptionWords threshold
+ */
+import { describe, expect, it } from 'vitest';
+import { splitWords } from '../tokenize/basic/word.js';
+
+describe('Interruption Detection - Word Counting', () => {
+  describe('Word Splitting Behavior', () => {
+    it('should count empty string as 0 words', () => {
+      const text = '';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(0);
+    });
+
+    it('should count single word correctly', () => {
+      const text = 'hello';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(1);
+    });
+
+    it('should count two words correctly', () => {
+      const text = 'hello world';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+
+    it('should count multiple words correctly', () => {
+      const text = 'hello this is a full sentence';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(6);
+    });
+
+    it('should handle punctuation correctly', () => {
+      const text = 'hello, world!';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+
+    it('should handle multiple spaces between words', () => {
+      const text = 'hello  world';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+
+    it('should count whitespace-only string as 0 words', () => {
+      const text = '   ';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(0);
+    });
+
+    it('should handle leading and trailing whitespace', () => {
+      const text = '  hello world  ';
+      const wordCount = splitWords(text, true).length;
+      expect(wordCount).toBe(2);
+    });
+  });
+
+  describe('Integration: Full Interruption Check Logic', () => {
+    it('should block interruption for empty transcript with threshold 2', () => {
+      const text = '';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('');
+      expect(wordCount).toBe(0);
+      expect(shouldBlock).toBe(true);
+    });
+
+    it('should block interruption for undefined transcript with threshold 2', () => {
+      const text: string | undefined = undefined;
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('');
+      expect(wordCount).toBe(0);
+      expect(shouldBlock).toBe(true);
+    });
+
+    it('should block interruption for single word with threshold 2', () => {
+      const text = 'hello';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('hello');
+      expect(wordCount).toBe(1);
+      expect(shouldBlock).toBe(true);
+    });
+
+    it('should allow interruption when word count exactly meets threshold', () => {
+      const text = 'hello world';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('hello world');
+      expect(wordCount).toBe(2);
+      expect(shouldBlock).toBe(false);
+    });
+
+    it('should allow interruption when word count exceeds threshold', () => {
+      const text = 'hello this is a full sentence';
+      const minInterruptionWords = 2;
+
+      const normalizedText = text ?? '';
+      const wordCount = splitWords(normalizedText, true).length;
+      const shouldBlock = wordCount < minInterruptionWords;
+
+      expect(normalizedText).toBe('hello this is a full sentence');
+      expect(wordCount).toBe(6);
+      expect(shouldBlock).toBe(false);
+    });
+
+    it('should apply consistent word counting logic in both methods', () => {
+      const transcripts = ['', 'hello', 'hello world', 'this is a longer sentence'];
+      const threshold = 2;
+
+      transcripts.forEach((transcript) => {
+        const text1 = transcript;
+        const normalizedText1 = text1 ?? '';
+        const wordCount1 = splitWords(normalizedText1, true).length;
+        const shouldBlock1 = wordCount1 < threshold;
+
+        const wordCount2 = splitWords(transcript, true).length;
+        const shouldBlock2 = wordCount2 < threshold;
+
+        expect(wordCount1).toBe(wordCount2);
+        expect(shouldBlock1).toBe(shouldBlock2);
+      });
+    });
+  });
+});

package/src/voice/io.ts

@@ -8,7 +8,7 @@ import type { ChatContext } from '../llm/chat_context.js';
 import type { ChatChunk } from '../llm/llm.js';
 import type { ToolContext } from '../llm/tool_context.js';
 import { log } from '../log.js';
-import { DeferredReadableStream } from '../stream/deferred_stream.js';
+import { MultiInputStream } from '../stream/multi_input_stream.js';
 import type { SpeechEvent } from '../stt/stt.js';
 import { Future } from '../utils.js';
 import type { ModelSettings } from './agent.js';
@@ -28,12 +28,70 @@ export type TTSNode = (
   text: ReadableStream<string>,
   modelSettings: ModelSettings,
 ) => Promise<ReadableStream<AudioFrame> | null>;
+
+/**
+ * Symbol used to identify TimedString objects.
+ */
+export const TIMED_STRING_SYMBOL = Symbol.for('lk.TimedString');
+
+/**
+ * A string with optional start and end timestamps for word-level alignment.
+ */
+export interface TimedString {
+  readonly [TIMED_STRING_SYMBOL]: true;
+  text: string;
+  startTime?: number; // seconds
+  endTime?: number; // seconds
+  confidence?: number;
+  startTimeOffset?: number;
+}
+
+/**
+ * Factory function to create a TimedString object.
+ */
+export function createTimedString(opts: {
+  text: string;
+  startTime?: number;
+  endTime?: number;
+  confidence?: number;
+  startTimeOffset?: number;
+}): TimedString {
+  return {
+    [TIMED_STRING_SYMBOL]: true,
+    text: opts.text,
+    startTime: opts.startTime,
+    endTime: opts.endTime,
+    confidence: opts.confidence,
+    startTimeOffset: opts.startTimeOffset,
+  };
+}
+
+/**
+ * Type guard to check if a value is a TimedString.
+ */
+export function isTimedString(value: unknown): value is TimedString {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    TIMED_STRING_SYMBOL in value &&
+    (value as TimedString)[TIMED_STRING_SYMBOL] === true
+  );
+}
+
+export interface AudioOutputCapabilities {
+  /** Whether this output supports pause/resume functionality */
+  pause: boolean;
+}
+
 export abstract class AudioInput {
-  protected deferredStream: DeferredReadableStream<AudioFrame> =
-    new DeferredReadableStream<AudioFrame>();
+  protected multiStream: MultiInputStream<AudioFrame> = new MultiInputStream<AudioFrame>();
 
   get stream(): ReadableStream<AudioFrame> {
-    return this.deferredStream.stream;
+    return this.multiStream.stream;
+  }
+
+  async close(): Promise<void> {
+    await this.multiStream.close();
   }
 
   onAttached(): void {}
@@ -42,6 +100,7 @@ export abstract class AudioInput {
 }
 
 export abstract class AudioOutput extends EventEmitter {
+  static readonly EVENT_PLAYBACK_STARTED = 'playbackStarted';
   static readonly EVENT_PLAYBACK_FINISHED = 'playbackFinished';
 
   private playbackFinishedFuture: Future<void> = new Future();
@@ -53,19 +112,33 @@ export abstract class AudioOutput extends EventEmitter {
     interrupted: false,
   };
   protected logger = log();
+  protected readonly capabilities: AudioOutputCapabilities;
 
   constructor(
     public sampleRate?: number,
     protected readonly nextInChain?: AudioOutput,
+    capabilities: AudioOutputCapabilities = { pause: false },
   ) {
     super();
+    this.capabilities = capabilities;
+
     if (this.nextInChain) {
+      this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, (ev: PlaybackStartedEvent) =>
+        this.onPlaybackStarted(ev.createdAt),
+      );
       this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) =>
         this.onPlaybackFinished(ev),
       );
     }
   }
 
+  /**
+   * Whether this output and all outputs in the chain support pause/resume.
+   */
+  get canPause(): boolean {
+    return this.capabilities.pause && (this.nextInChain?.canPause ?? true);
+  }
+
   /**
    * Capture an audio frame for playback, frames can be pushed faster than real-time
    */
@@ -92,6 +165,14 @@ export abstract class AudioOutput extends EventEmitter {
     return this.lastPlaybackEvent;
   }
 
+  /**
+   * Called when playback actually starts (first frame is sent to output).
+   * Developers building audio sinks should call this when the first frame is captured.
+   */
+  onPlaybackStarted(createdAt: number): void {
+    this.emit(AudioOutput.EVENT_PLAYBACK_STARTED, { createdAt } as PlaybackStartedEvent);
+  }
+
   /**
    * Developers building audio sinks must call this method when a playback/segment is finished.
    * Segments are segmented by calls to flush() or clearBuffer()
@@ -128,25 +209,47 @@ export abstract class AudioOutput extends EventEmitter {
       this.nextInChain.onDetached();
     }
   }
+
+  /**
+   * Pause the audio playback
+   */
+  pause(): void {
+    if (this.nextInChain) {
+      this.nextInChain.pause();
+    }
+  }
+
+  /**
+   * Resume the audio playback
+   */
+  resume(): void {
+    if (this.nextInChain) {
+      this.nextInChain.resume();
+    }
+  }
 }
 
 export interface PlaybackFinishedEvent {
-  // How much of the audio was played back
+  /** How much of the audio was played back, in seconds */
   playbackPosition: number;
-  // Interrupted is True if playback was interrupted (clearBuffer() was called)
+  /** True if playback was interrupted (clearBuffer() was called) */
   interrupted: boolean;
-  // Transcript synced with playback; may be partial if the audio was interrupted
-  // When null, the transcript is not synchronized with the playback
+  /**
+   * Transcript synced with playback; may be partial if the audio was interrupted.
+   * When undefined, the transcript is not synchronized with the playback.
+   */
   synchronizedTranscript?: string;
 }
 
+export interface PlaybackStartedEvent {
+  /** The timestamp (Date.now()) when the playback started */
+  createdAt: number;
+}
+
 export abstract class TextOutput {
   constructor(protected readonly nextInChain?: TextOutput) {}
 
-  /**
-   * Capture a text segment (Used by the output of LLM nodes)
-   */
-  abstract captureText(text: string): Promise<void>;
+  abstract captureText(text: string | TimedString): Promise<void>;
 
   /**
    * Mark the current text segment as complete (e.g LLM generation is complete)

package/src/voice/recorder_io/index.ts

@@ -0,0 +1,4 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './recorder_io.js';