@livekit/agents 1.1.0 → 1.2.0

package/src/voice/room_io/room_io.ts

@@ -1,17 +1,18 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { TextStreamReader } from '@livekit/rtc-node';
 import {
+  type AudioFrame,
   ConnectionState,
   DisconnectReason,
+  type FrameProcessor,
   type NoiseCancellationOptions,
   type Participant,
   ParticipantKind,
   type RemoteParticipant,
   type Room,
   RoomEvent,
-  type TextStreamInfo,
-  type TextStreamReader,
   TrackPublishOptions,
   TrackSource,
 } from '@livekit/rtc-node';
@@ -19,7 +20,7 @@ import type { WritableStreamDefaultWriter } from 'node:stream/web';
 import { ATTRIBUTE_PUBLISH_ON_BEHALF, TOPIC_CHAT } from '../../constants.js';
 import { log } from '../../log.js';
 import { IdentityTransform } from '../../stream/identity_transform.js';
-import { Future, Task } from '../../utils.js';
+import { Future, Task, waitForAbort } from '../../utils.js';
 import { type AgentSession } from '../agent_session.js';
 import {
   AgentSessionEventTypes,
@@ -28,6 +29,7 @@ import {
   type UserInputTranscribedEvent,
 } from '../events.js';
 import type { AudioOutput, TextOutput } from '../io.js';
+import type { TextInputCallback } from '../remote_session.js';
 import { TranscriptionSynchronizer } from '../transcription/synchronizer.js';
 import { ParticipantAudioInputStream } from './_input.js';
 import {
@@ -37,20 +39,13 @@ import {
   ParticipantTranscriptionOutput,
 } from './_output.js';
 
-export interface TextInputEvent {
-  text: string;
-  info: TextStreamInfo;
-  participant: RemoteParticipant;
-}
-
-export type TextInputCallback = (sess: AgentSession, ev: TextInputEvent) => void | Promise<void>;
-
-const DEFAULT_TEXT_INPUT_CALLBACK: TextInputCallback = (sess: AgentSession, ev: TextInputEvent) => {
+export const DEFAULT_TEXT_INPUT_CALLBACK: TextInputCallback = (sess, ev) => {
   sess.interrupt();
   sess.generateReply({ userInput: ev.text });
 };
 
 const DEFAULT_PARTICIPANT_KINDS: ParticipantKind[] = [
+  ParticipantKind.CONNECTOR,
   ParticipantKind.SIP,
   ParticipantKind.STANDARD,
 ];
@@ -64,22 +59,41 @@ const CLOSE_ON_DISCONNECT_REASONS: DisconnectReason[] = [
 export interface RoomInputOptions {
   audioSampleRate: number;
   audioNumChannels: number;
+  /** If not given, default to True. */
   textEnabled: boolean;
+  /** If not given, default to True. */
   audioEnabled: boolean;
+  /** If not given, default to False. */
   videoEnabled: boolean;
+  /** The participant to link to. If not provided, link to the first participant.
+    Can be overridden by the `participant` argument of RoomIO constructor or `set_participant`.
+  */
   participantIdentity?: string;
-  noiseCancellation?: NoiseCancellationOptions;
+  noiseCancellation?: NoiseCancellationOptions | FrameProcessor<AudioFrame>;
   textInputCallback?: TextInputCallback;
+  /** Participant kinds accepted for auto subscription. If not provided,
+    accept `DEFAULT_PARTICIPANT_KINDS`
+  */
   participantKinds?: ParticipantKind[];
+  /** Close the AgentSession if the linked participant disconnects with reasons in
+    CLIENT_INITIATED, ROOM_DELETED, or USER_REJECTED.
+  */
   closeOnDisconnect: boolean;
 }
 
 export interface RoomOutputOptions {
+  /** If not given, default to True. */
   transcriptionEnabled: boolean;
+  /** If not given, default to True. */
   audioEnabled: boolean;
   audioSampleRate: number;
   audioNumChannels: number;
+  /** False to disable transcription synchronization with audio output.
+    Otherwise, transcription is emitted as quickly as available.
+  */
   syncTranscription: boolean;
+  /** The name of the audio track to publish. If not provided, default to "roomio_audio".
+   */
   audioPublishOptions: TrackPublishOptions;
 }
 
@@ -114,6 +128,7 @@ export class RoomIO {
   private agentTranscriptOutput?: ParalellTextOutput;
   private transcriptionSynchronizer?: TranscriptionSynchronizer;
   private participantIdentity: string | null = null;
+  private textStreamHandlerRegistered = false;
 
   private participantAvailableFuture: Future<RemoteParticipant> = new Future();
   private roomConnectedFuture: Future<void> = new Future();
@@ -124,8 +139,6 @@ export class RoomIO {
   private forwardUserTranscriptTask?: Task<void>;
   private initTask?: Task<void>;
 
-  private textStreamHandlerRegistered = false;
-
   private logger = log();
 
   constructor({
@@ -155,7 +168,10 @@ export class RoomIO {
       : this.inputOptions.participantIdentity ?? null;
   }
   private async init(signal: AbortSignal): Promise<void> {
-    await this.roomConnectedFuture.await;
+    await Promise.race([this.roomConnectedFuture.await, waitForAbort(signal)]);
+    if (signal.aborted) {
+      return;
+    }
 
     for (const participant of this.room.remoteParticipants.values()) {
       this.onParticipantConnected(participant);
@@ -164,7 +180,15 @@ export class RoomIO {
       return;
     }
 
-    const participant = await this.participantAvailableFuture.await;
+    const participant = await Promise.race([
+      this.participantAvailableFuture.await,
+      waitForAbort(signal),
+    ]);
+
+    if (!participant) {
+      return;
+    }
+
     this.setParticipant(participant.identity);
 
     // init agent outputs
@@ -250,7 +274,7 @@ export class RoomIO {
   };
 
   private onUserTextInput = (reader: TextStreamReader, participantInfo: { identity: string }) => {
-    if (participantInfo.identity !== this.participantIdentity) {
+    if (this.participantIdentity && participantInfo.identity !== this.participantIdentity) {
       return;
     }
 
@@ -266,7 +290,7 @@ export class RoomIO {
       const textInputResult = this.inputOptions.textInputCallback!(this.agentSession, {
         text,
         info: reader.info,
-        participant,
+        participantIdentity: participantInfo.identity,
       });
 
       // check if callback is a Promise
@@ -350,7 +374,27 @@ export class RoomIO {
     return this.transcriptionSynchronizer.textOutput;
   }
 
-  /* Switch to a different participant */
+  get isParticipantAvailable(): boolean {
+    return this.participantAvailableFuture.done;
+  }
+
+  get rtcRoom(): Room {
+    return this.room;
+  }
+
+  get linkedParticipant(): RemoteParticipant | undefined {
+    if (!this.isParticipantAvailable) {
+      return undefined;
+    }
+
+    return this.participantAvailableFuture.result;
+  }
+
+  get localParticipant(): Participant | undefined {
+    return this.room.localParticipant ?? undefined;
+  }
+
+  /** Switch to a different participant */
   setParticipant(participantIdentity: string | null) {
     this.logger.debug({ participantIdentity }, 'setting participant');
     if (participantIdentity === null) {
@@ -390,6 +434,8 @@ export class RoomIO {
   }
 
   start() {
+    // -- create inputs --
+
     if (this.inputOptions.textEnabled) {
       try {
         this.room.registerTextStreamHandler(TOPIC_CHAT, this.onUserTextInput);
@@ -401,7 +447,6 @@ export class RoomIO {
       }
     }
 
-    // -- create inputs --
     if (this.inputOptions.audioEnabled) {
       this.audioInput = new ParticipantAudioInputStream({
         room: this.room,
@@ -454,7 +499,7 @@ export class RoomIO {
 
     this.initTask = Task.from((controller) => this.init(controller.signal));
 
-    // -- attatch the agent to the session --
+    // attach the agent to the session
     if (this.audioInput) {
       this.agentSession.input.audio = this.audioInput;
     }

package/src/voice/speech_handle.ts

@@ -1,10 +1,26 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { Context } from '@opentelemetry/api';
 import type { ChatItem } from '../llm/index.js';
-import { Event, Future, shortuuid } from '../utils.js';
 import type { Task } from '../utils.js';
-import { asyncLocalStorage } from './agent.js';
+import { Event, Future, shortuuid } from '../utils.js';
+import { functionCallStorage } from './agent.js';
+
+/** Symbol used to identify SpeechHandle instances */
+const SPEECH_HANDLE_SYMBOL = Symbol.for('livekit.agents.SpeechHandle');
+
+/**
+ * Type guard to check if a value is a SpeechHandle.
+ */
+export function isSpeechHandle(value: unknown): value is SpeechHandle {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    SPEECH_HANDLE_SYMBOL in value &&
+    (value as Record<symbol, boolean>)[SPEECH_HANDLE_SYMBOL] === true
+  );
+}
 
 export class SpeechHandle {
   /** Priority for messages that should be played after all other messages in the queue */
@@ -18,16 +34,27 @@ export class SpeechHandle {
   private authorizedEvent = new Event();
   private scheduledFut = new Future<void>();
   private doneFut = new Future<void>();
-
   private generations: Future<void>[] = [];
+  private _chatItems: ChatItem[] = [];
+
   /** @internal */
   _tasks: Task<void>[] = [];
-  private _chatItems: ChatItem[] = [];
-  private _numSteps = 1;
+
+  /** @internal */
+  _numSteps = 1;
+
+  /** @internal - OpenTelemetry context for the agent turn span */
+  _agentTurnContext?: Context;
+
+  /** @internal - used by AgentTask/RunResult final output plumbing */
+  _maybeRunFinalOutput?: unknown;
 
   private itemAddedCallbacks: Set<(item: ChatItem) => void> = new Set();
   private doneCallbacks: Set<(sh: SpeechHandle) => void> = new Set();
 
+  /** @internal Symbol marker for type identification */
+  readonly [SPEECH_HANDLE_SYMBOL] = true;
+
   constructor(
     private _id: string,
     private _allowInterruptions: boolean,
@@ -124,7 +151,7 @@ export class SpeechHandle {
    * has entirely played out, including any tool calls and response follow-ups.
    */
   async waitForPlayout(): Promise<void> {
-    const store = asyncLocalStorage.getStore();
+    const store = functionCallStorage.getStore();
     if (store && store?.functionCall) {
       throw new Error(
         `Cannot call 'SpeechHandle.waitForPlayout()' from inside the function tool '${store.functionCall.name}'. ` +
@@ -133,6 +160,7 @@ export class SpeechHandle {
           "To wait for the assistant's spoken response prior to running this tool, use RunContext.wait_for_playout() instead.",
       );
     }
+    await this.doneFut.await;
   }
 
   async waitIfNotInterrupted(aw: Promise<unknown>[]): Promise<void> {
@@ -142,6 +170,10 @@ export class SpeechHandle {
   }
 
   addDoneCallback(callback: (sh: SpeechHandle) => void) {
+    if (this.done()) {
+      queueMicrotask(() => callback(this));
+      return;
+    }
     this.doneCallbacks.add(callback);
   }
 

package/src/voice/testing/fake_llm.ts

@@ -0,0 +1,138 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext } from '../../llm/chat_context.js';
+import { FunctionCall } from '../../llm/chat_context.js';
+import { LLMStream as BaseLLMStream, LLM, type LLMStream } from '../../llm/llm.js';
+import type { ToolChoice, ToolContext } from '../../llm/tool_context.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../../types.js';
+import { delay } from '../../utils.js';
+
+export interface FakeLLMResponse {
+  input: string;
+  type?: 'llm';
+  content?: string;
+  ttft?: number;
+  duration?: number;
+  toolCalls?: Array<{ name: string; args: Record<string, unknown> }>;
+}
+
+export class FakeLLM extends LLM {
+  private readonly responseMap = new Map<string, FakeLLMResponse>();
+
+  constructor(responses: FakeLLMResponse[] = []) {
+    super();
+    for (const response of responses) {
+      this.responseMap.set(response.input, {
+        type: 'llm',
+        ttft: 0,
+        duration: 0,
+        ...response,
+      });
+    }
+  }
+
+  label(): string {
+    return 'fake-llm';
+  }
+
+  chat({
+    chatCtx,
+    toolCtx,
+    connOptions = DEFAULT_API_CONNECT_OPTIONS,
+  }: {
+    chatCtx: ChatContext;
+    toolCtx?: ToolContext;
+    connOptions?: APIConnectOptions;
+    parallelToolCalls?: boolean;
+    toolChoice?: ToolChoice;
+    extraKwargs?: Record<string, unknown>;
+  }): LLMStream {
+    return new FakeLLMStream(this, {
+      chatCtx,
+      toolCtx,
+      connOptions,
+    });
+  }
+
+  lookup(input: string): FakeLLMResponse | undefined {
+    return this.responseMap.get(input);
+  }
+}
+
+class FakeLLMStream extends BaseLLMStream {
+  private readonly fake: FakeLLM;
+
+  constructor(
+    fake: FakeLLM,
+    params: { chatCtx: ChatContext; toolCtx?: ToolContext; connOptions: APIConnectOptions },
+  ) {
+    super(fake, params);
+    this.fake = fake;
+  }
+
+  protected async run(): Promise<void> {
+    const input = this.getInputText();
+    const decision = this.fake.lookup(input);
+    if (!decision) {
+      return;
+    }
+
+    const startedAt = Date.now();
+    if ((decision.ttft ?? 0) > 0) {
+      await delay(decision.ttft!);
+    }
+
+    const content = decision.content ?? '';
+    const chunkSize = 3;
+    for (let i = 0; i < content.length; i += chunkSize) {
+      this.queue.put({
+        id: 'fake',
+        delta: { role: 'assistant', content: content.slice(i, i + chunkSize) },
+      });
+    }
+
+    if (decision.toolCalls && decision.toolCalls.length > 0) {
+      const calls = decision.toolCalls.map((tc, index) =>
+        FunctionCall.create({
+          callId: `fake_call_${index}`,
+          name: tc.name,
+          args: JSON.stringify(tc.args),
+        }),
+      );
+      this.queue.put({
+        id: 'fake',
+        delta: { role: 'assistant', toolCalls: calls },
+      });
+    }
+
+    const elapsed = Date.now() - startedAt;
+    const waitMs = Math.max(0, (decision.duration ?? 0) - elapsed);
+    if (waitMs > 0) {
+      await delay(waitMs);
+    }
+  }
+
+  private getInputText(): string {
+    const items = this.chatCtx.items;
+    if (items.length === 0) {
+      throw new Error('No input text found');
+    }
+
+    for (const item of items) {
+      if (item.type === 'message' && item.role === 'system') {
+        const text = item.textContent ?? '';
+        const lines = text.split('\n');
+        const tail = lines[lines.length - 1] ?? '';
+        if (lines.length > 1 && tail.startsWith('instructions:')) {
+          return tail;
+        }
+      }
+    }
+
+    const last = items[items.length - 1]!;
+    if (last.type === 'message' && last.role === 'user') return last.textContent ?? '';
+    if (last.type === 'function_call_output') return last.output;
+    throw new Error('No input text found');
+  }
+}

package/src/voice/testing/index.ts

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Testing utilities for agent evaluation.
+ *
+ * @example
+ * ```typescript
+ * import { AgentSession, Agent, voice } from '@livekit/agents';
+ *
+ * const session = new AgentSession({ llm });
+ * await session.start(agent);
+ *
+ * const result = await session.run({ userInput: 'Hello' });
+ * result.expect.nextEvent().isMessage({ role: 'assistant' });
+ * result.expect.noMoreEvents();
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+export {
+  AgentHandoffAssert,
+  AssertionError,
+  EventAssert,
+  EventRangeAssert,
+  FunctionCallAssert,
+  FunctionCallOutputAssert,
+  MessageAssert,
+  RunAssert,
+  RunResult,
+} from './run_result.js';
+
+export {
+  isAgentHandoffEvent,
+  isChatMessageEvent,
+  isFunctionCallEvent,
+  isFunctionCallOutputEvent,
+  type AgentHandoffAssertOptions,
+  type AgentHandoffEvent,
+  type ChatMessageEvent,
+  type EventType,
+  type FunctionCallAssertOptions,
+  type FunctionCallEvent,
+  type FunctionCallOutputAssertOptions,
+  type FunctionCallOutputEvent,
+  type MessageAssertOptions,
+  type RunEvent,
+} from './types.js';
+
+export { FakeLLM, type FakeLLMResponse } from './fake_llm.js';