@livekit/agents 1.0.45 → 1.0.47

package/src/telemetry/trace_types.ts

@@ -20,6 +20,8 @@ export const ATTR_ROOM_NAME = 'lk.room_name';
 export const ATTR_SESSION_OPTIONS = 'lk.session_options';
 
 // assistant turn
+export const ATTR_AGENT_TURN_ID = 'lk.generation_id';
+export const ATTR_AGENT_PARENT_TURN_ID = 'lk.parent_generation_id';
 export const ATTR_USER_INPUT = 'lk.user_input';
 export const ATTR_INSTRUCTIONS = 'lk.instructions';
 export const ATTR_SPEECH_INTERRUPTED = 'lk.interrupted';
@@ -27,10 +29,14 @@ export const ATTR_SPEECH_INTERRUPTED = 'lk.interrupted';
 // llm node
 export const ATTR_CHAT_CTX = 'lk.chat_ctx';
 export const ATTR_FUNCTION_TOOLS = 'lk.function_tools';
+export const ATTR_PROVIDER_TOOLS = 'lk.provider_tools';
+export const ATTR_TOOL_SETS = 'lk.tool_sets';
 export const ATTR_RESPONSE_TEXT = 'lk.response.text';
 export const ATTR_RESPONSE_FUNCTION_CALLS = 'lk.response.function_calls';
+export const ATTR_RESPONSE_TTFT = 'lk.response.ttft';
 
 // function tool
+export const ATTR_FUNCTION_TOOL_ID = 'lk.function_tool.id';
 export const ATTR_FUNCTION_TOOL_NAME = 'lk.function_tool.name';
 export const ATTR_FUNCTION_TOOL_ARGS = 'lk.function_tool.arguments';
 export const ATTR_FUNCTION_TOOL_IS_ERROR = 'lk.function_tool.is_error';
@@ -40,6 +46,7 @@ export const ATTR_FUNCTION_TOOL_OUTPUT = 'lk.function_tool.output';
 export const ATTR_TTS_INPUT_TEXT = 'lk.input_text';
 export const ATTR_TTS_STREAMING = 'lk.tts.streaming';
 export const ATTR_TTS_LABEL = 'lk.tts.label';
+export const ATTR_RESPONSE_TTFB = 'lk.response.ttfb';
 
 // eou detection
 export const ATTR_EOU_PROBABILITY = 'lk.eou.probability';
@@ -56,10 +63,14 @@ export const ATTR_LLM_METRICS = 'lk.llm_metrics';
 export const ATTR_TTS_METRICS = 'lk.tts_metrics';
 export const ATTR_REALTIME_MODEL_METRICS = 'lk.realtime_model_metrics';
 
+// latency span attributes
+export const ATTR_E2E_LATENCY = 'lk.e2e_latency';
+
 // OpenTelemetry GenAI attributes
 // OpenTelemetry specification: https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/
 export const ATTR_GEN_AI_OPERATION_NAME = 'gen_ai.operation.name';
 export const ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model';
+export const ATTR_GEN_AI_PROVIDER_NAME = 'gen_ai.provider.name';
 export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens';
 export const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens';
 
@@ -86,3 +97,10 @@ export const ATTR_EXCEPTION_MESSAGE = 'exception.message';
 
 // Platform-specific attributes
 export const ATTR_LANGFUSE_COMPLETION_START_TIME = 'langfuse.observation.completion_start_time';
+
+// Adaptive Interruption attributes
+export const ATTR_IS_INTERRUPTION = 'lk.is_interruption';
+export const ATTR_INTERRUPTION_PROBABILITY = 'lk.interruption.probability';
+export const ATTR_INTERRUPTION_TOTAL_DURATION = 'lk.interruption.total_duration';
+export const ATTR_INTERRUPTION_PREDICTION_DURATION = 'lk.interruption.prediction_duration';
+export const ATTR_INTERRUPTION_DETECTION_DELAY = 'lk.interruption.detection_delay';

package/src/utils.test.ts

@@ -469,6 +469,93 @@ describe('utils', () => {
         expect((error as Error).name).toBe('TypeError');
       }
     });
+
+    it('should return undefined for Task.current outside task context', () => {
+      expect(Task.current()).toBeUndefined();
+    });
+
+    it('should preserve Task.current inside a task across awaits', async () => {
+      const task = Task.from(
+        async () => {
+          const currentAtStart = Task.current();
+          await delay(5);
+          const currentAfterAwait = Task.current();
+
+          expect(currentAtStart).toBeDefined();
+          expect(currentAfterAwait).toBe(currentAtStart);
+
+          return currentAtStart;
+        },
+        undefined,
+        'current-context-test',
+      );
+
+      const currentFromResult = await task.result;
+      expect(currentFromResult).toBe(task);
+    });
+
+    it('should isolate nested Task.current context and restore parent context', async () => {
+      const parentTask = Task.from(
+        async (controller) => {
+          const parentCurrent = Task.current();
+          expect(parentCurrent).toBeDefined();
+
+          const childTask = Task.from(
+            async () => {
+              const childCurrentStart = Task.current();
+              await delay(5);
+              const childCurrentAfterAwait = Task.current();
+
+              expect(childCurrentStart).toBeDefined();
+              expect(childCurrentAfterAwait).toBe(childCurrentStart);
+              expect(childCurrentStart).not.toBe(parentCurrent);
+
+              return childCurrentStart;
+            },
+            controller,
+            'child-current-context-test',
+          );
+
+          const childCurrent = await childTask.result;
+          const parentCurrentAfterChild = Task.current();
+
+          expect(parentCurrentAfterChild).toBe(parentCurrent);
+
+          return { parentCurrent, childCurrent };
+        },
+        undefined,
+        'parent-current-context-test',
+      );
+
+      const { parentCurrent, childCurrent } = await parentTask.result;
+      expect(parentCurrent).toBe(parentTask);
+      expect(childCurrent).not.toBe(parentCurrent);
+      expect(Task.current()).toBeUndefined();
+    });
+
+    it('should always expose Task.current for concurrent task callbacks', async () => {
+      const tasks = Array.from({ length: 25 }, (_, idx) =>
+        Task.from(
+          async () => {
+            const currentAtStart = Task.current();
+            await delay(1);
+            const currentAfterAwait = Task.current();
+
+            expect(currentAtStart).toBeDefined();
+            expect(currentAfterAwait).toBe(currentAtStart);
+
+            return currentAtStart;
+          },
+          undefined,
+          `current-context-stress-${idx}`,
+        ),
+      );
+
+      const currentTasks = await Promise.all(tasks.map((task) => task.result));
+      currentTasks.forEach((currentTask, idx) => {
+        expect(currentTask).toBe(tasks[idx]);
+      });
+    });
   });
 
   describe('Event', () => {

package/src/utils.ts

@@ -9,6 +9,7 @@ import type {
   TrackKind,
 } from '@livekit/rtc-node';
 import { AudioFrame, AudioResampler, RoomEvent } from '@livekit/rtc-node';
+import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter, once } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
 import { TransformStream, type TransformStreamDefaultController } from 'node:stream/web';
@@ -126,6 +127,8 @@ export class Future<T = void> {
   #rejectPromise!: (error: Error) => void;
   #done: boolean = false;
   #rejected: boolean = false;
+  #result: T | undefined = undefined;
+  #error: Error | undefined = undefined;
 
   constructor() {
     this.#await = new Promise<T>((resolve, reject) => {
@@ -142,6 +145,18 @@ export class Future<T = void> {
     return this.#done;
   }
 
+  get result(): T {
+    if (!this.#done) {
+      throw new Error('Future is not done');
+    }
+
+    if (this.#rejected) {
+      throw this.#error;
+    }
+
+    return this.#result!;
+  }
+
   /** Whether the future was rejected (cancelled) */
   get rejected() {
     return this.#rejected;
@@ -149,12 +164,14 @@ export class Future<T = void> {
 
   resolve(value: T) {
     this.#done = true;
+    this.#result = value;
     this.#resolvePromise(value);
   }
 
   reject(error: Error) {
     this.#done = true;
     this.#rejected = true;
+    this.#error = error;
     this.#rejectPromise(error);
   }
 }
@@ -418,7 +435,9 @@ export enum TaskResult {
  * @param T - The type of the task result
  */
 export class Task<T> {
+  private static readonly currentTaskStorage = new AsyncLocalStorage<Task<unknown>>();
   private resultFuture: Future<T>;
+  private doneCallbacks: Set<() => void> = new Set();
 
   #logger = log();
 
@@ -428,6 +447,21 @@ export class Task<T> {
     readonly name?: string,
   ) {
     this.resultFuture = new Future();
+    void this.resultFuture.await
+      .then(
+        () => undefined,
+        () => undefined,
+      )
+      .finally(() => {
+        for (const callback of this.doneCallbacks) {
+          try {
+            callback();
+          } catch (error) {
+            this.#logger.error({ error }, 'Task done callback failed');
+          }
+        }
+        this.doneCallbacks.clear();
+      });
     this.runTask();
   }
 
@@ -447,6 +481,13 @@ export class Task<T> {
     return new Task(fn, abortController, name);
   }
 
+  /**
+   * Returns the currently running task in this async context, if available.
+   */
+  static current(): Task<unknown> | undefined {
+    return Task.currentTaskStorage.getStore();
+  }
+
   private async runTask() {
     const run = async () => {
       if (this.name) {
@@ -455,7 +496,8 @@ export class Task<T> {
       return await this.fn(this.controller);
     };
 
-    return run()
+    return Task.currentTaskStorage
+      .run(this as Task<unknown>, run)
       .then((value) => {
         this.resultFuture.resolve(value);
         return value;
@@ -527,7 +569,15 @@ export class Task<T> {
   }
 
   addDoneCallback(callback: () => void) {
-    this.resultFuture.await.finally(callback);
+    if (this.done) {
+      queueMicrotask(callback);
+      return;
+    }
+    this.doneCallbacks.add(callback);
+  }
+
+  removeDoneCallback(callback: () => void) {
+    this.doneCallbacks.delete(callback);
   }
 }
 

package/src/version.ts

@@ -2,4 +2,4 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-export const version = '0.1.0';
+export const version = __PACKAGE_VERSION__;

package/src/voice/agent.test.ts

@@ -1,10 +1,15 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 import { z } from 'zod';
 import { tool } from '../llm/index.js';
-import { Agent } from './agent.js';
+import { initializeLogger } from '../log.js';
+import { Task } from '../utils.js';
+import { Agent, AgentTask, _setActivityTaskInfo } from './agent.js';
+import { agentActivityStorage } from './agent_activity.js';
+
+initializeLogger({ pretty: false, level: 'error' });
 
 describe('Agent', () => {
   it('should create agent with basic instructions', () => {
@@ -77,4 +82,137 @@ describe('Agent', () => {
     expect(tools1).toEqual(tools2);
     expect(tools1).toEqual(tools);
   });
+
+  it('should require AgentTask to run inside task context', async () => {
+    class TestTask extends AgentTask<string> {
+      constructor() {
+        super({ instructions: 'test task' });
+      }
+    }
+
+    const task = new TestTask();
+    await expect(task.run()).rejects.toThrow('must be executed inside a Task context');
+  });
+
+  it('should require AgentTask to run inside inline task context', async () => {
+    class TestTask extends AgentTask<string> {
+      constructor() {
+        super({ instructions: 'test task' });
+      }
+    }
+
+    const task = new TestTask();
+    const wrapper = Task.from(async () => {
+      return await task.run();
+    });
+
+    await expect(wrapper.result).rejects.toThrow(
+      'should only be awaited inside function tools or the onEnter/onExit methods of an Agent',
+    );
+  });
+
+  it('should allow AgentTask run from inline task context', async () => {
+    class TestTask extends AgentTask<string> {
+      constructor() {
+        super({ instructions: 'test task' });
+      }
+    }
+
+    const task = new TestTask();
+    const oldAgent = new Agent({ instructions: 'old agent' });
+    const mockSession = {
+      currentAgent: oldAgent,
+      _globalRunState: undefined,
+      _updateActivity: async (agent: Agent) => {
+        if (agent === task) {
+          task.complete('ok');
+        }
+      },
+    };
+
+    const mockActivity = {
+      agent: oldAgent,
+      agentSession: mockSession,
+      _onEnterTask: undefined,
+      llm: undefined,
+      close: async () => {},
+    };
+
+    const wrapper = Task.from(async () => {
+      const currentTask = Task.current();
+      if (!currentTask) {
+        throw new Error('expected task context');
+      }
+      _setActivityTaskInfo(currentTask, { inlineTask: true });
+      return await agentActivityStorage.run(mockActivity as any, () => task.run());
+    });
+
+    await expect(wrapper.result).resolves.toBe('ok');
+  });
+
+  it('should require AgentTask to run inside AgentActivity context', async () => {
+    class TestTask extends AgentTask<string> {
+      constructor() {
+        super({ instructions: 'test task' });
+      }
+    }
+
+    const task = new TestTask();
+    const wrapper = Task.from(async () => {
+      const currentTask = Task.current();
+      if (!currentTask) {
+        throw new Error('expected task context');
+      }
+      _setActivityTaskInfo(currentTask, { inlineTask: true });
+      return await task.run();
+    });
+
+    await expect(wrapper.result).rejects.toThrow(
+      'must be executed inside an AgentActivity context',
+    );
+  });
+
+  it('should close old activity when current agent changes while AgentTask is pending', async () => {
+    class TestTask extends AgentTask<string> {
+      constructor() {
+        super({ instructions: 'test task' });
+      }
+    }
+
+    const task = new TestTask();
+    const oldAgent = new Agent({ instructions: 'old agent' });
+    const switchedAgent = new Agent({ instructions: 'switched agent' });
+    const closeOldActivity = vi.fn(async () => {});
+
+    const mockSession = {
+      currentAgent: oldAgent as Agent,
+      _globalRunState: undefined,
+      _updateActivity: async (agent: Agent) => {
+        if (agent === task) {
+          mockSession.currentAgent = switchedAgent;
+          task.complete('ok');
+        }
+      },
+    };
+
+    const mockActivity = {
+      agent: oldAgent,
+      agentSession: mockSession,
+      _onEnterTask: undefined,
+      llm: undefined,
+      close: closeOldActivity,
+    };
+
+    const wrapper = Task.from(async () => {
+      const currentTask = Task.current();
+      if (!currentTask) {
+        throw new Error('expected task context');
+      }
+      _setActivityTaskInfo(currentTask, { inlineTask: true });
+      return await agentActivityStorage.run(mockActivity as any, () => task.run());
+    });
+
+    await expect(wrapper.result).resolves.toBe('ok');
+    expect(closeOldActivity).toHaveBeenCalledTimes(1);
+  });
 });

package/src/voice/agent.ts

@@ -13,26 +13,71 @@ import {
   type TTSModelString,
 } from '../inference/index.js';
 import { ReadonlyChatContext } from '../llm/chat_context.js';
-import type { ChatMessage, FunctionCall, RealtimeModel } from '../llm/index.js';
+import type { ChatMessage, FunctionCall } from '../llm/index.js';
 import {
   type ChatChunk,
   ChatContext,
   LLM,
+  RealtimeModel,
   type ToolChoice,
   type ToolContext,
 } from '../llm/index.js';
+import { log } from '../log.js';
 import type { STT, SpeechEvent } from '../stt/index.js';
 import { StreamAdapter as STTStreamAdapter } from '../stt/index.js';
 import { SentenceTokenizer as BasicSentenceTokenizer } from '../tokenize/basic/index.js';
 import type { TTS } from '../tts/index.js';
 import { SynthesizeStream, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
 import { USERDATA_TIMED_TRANSCRIPT } from '../types.js';
+import { Future, Task } from '../utils.js';
 import type { VAD } from '../vad.js';
-import type { AgentActivity } from './agent_activity.js';
+import { type AgentActivity, agentActivityStorage } from './agent_activity.js';
 import type { AgentSession, TurnDetectionMode } from './agent_session.js';
 import type { TimedString } from './io.js';
+import type { SpeechHandle } from './speech_handle.js';
+
+export const functionCallStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();
+export const speechHandleStorage = new AsyncLocalStorage<SpeechHandle>();
+const activityTaskInfoStorage = new WeakMap<Task<any>, _ActivityTaskInfo>();
+
+type _ActivityTaskInfo = {
+  functionCall: FunctionCall | null;
+  speechHandle: SpeechHandle | null;
+  inlineTask: boolean;
+};
+
+/** @internal */
+export function _setActivityTaskInfo<T>(
+  task: Task<T>,
+  options: {
+    functionCall?: FunctionCall | null;
+    speechHandle?: SpeechHandle | null;
+    inlineTask?: boolean;
+  },
+): void {
+  const info = activityTaskInfoStorage.get(task) ?? {
+    functionCall: null,
+    speechHandle: null,
+    inlineTask: false,
+  };
+
+  if (Object.hasOwn(options, 'functionCall')) {
+    info.functionCall = options.functionCall ?? null;
+  }
+  if (Object.hasOwn(options, 'speechHandle')) {
+    info.speechHandle = options.speechHandle ?? null;
+  }
+  if (Object.hasOwn(options, 'inlineTask')) {
+    info.inlineTask = options.inlineTask ?? false;
+  }
+
+  activityTaskInfoStorage.set(task, info);
+}
 
-export const asyncLocalStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();
+/** @internal */
+export function _getActivityTaskInfo<T>(task: Task<T>): _ActivityTaskInfo | undefined {
+  return activityTaskInfoStorage.get(task);
+}
 export const STOP_RESPONSE_SYMBOL = Symbol('StopResponse');
 
 export class StopResponse extends Error {
@@ -268,20 +313,20 @@ export class Agent<UserData = any> {
         throw new Error('sttNode called but no STT node is available');
       }
 
-      let wrapped_stt = activity.stt;
+      let wrappedStt = activity.stt;
 
-      if (!wrapped_stt.capabilities.streaming) {
+      if (!wrappedStt.capabilities.streaming) {
         const vad = agent.vad || activity.vad;
         if (!vad) {
           throw new Error(
             'STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming',
           );
         }
-        wrapped_stt = new STTStreamAdapter(wrapped_stt, vad);
+        wrappedStt = new STTStreamAdapter(wrappedStt, vad);
       }
 
       const connOptions = activity.agentSession.connOptions.sttConnOptions;
-      const stream = wrapped_stt.stream({ connOptions });
+      const stream = wrappedStt.stream({ connOptions });
 
       // Set startTimeOffset to provide linear timestamps across reconnections
       const audioInputStartedAt =
@@ -382,14 +427,14 @@ export class Agent<UserData = any> {
         throw new Error('ttsNode called but no TTS node is available');
       }
 
-      let wrapped_tts = activity.tts;
+      let wrappedTts = activity.tts;
 
       if (!activity.tts.capabilities.streaming) {
-        wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());
+        wrappedTts = new TTSStreamAdapter(wrappedTts, new BasicSentenceTokenizer());
       }
 
       const connOptions = activity.agentSession.connOptions.ttsConnOptions;
-      const stream = wrapped_tts.stream({ connOptions });
+      const stream = wrappedTts.stream({ connOptions });
       stream.updateInputStream(text);
 
       let cleaned = false;
@@ -440,3 +485,137 @@ export class Agent<UserData = any> {
     },
   };
 }
+
+export class AgentTask<ResultT = unknown, UserData = any> extends Agent<UserData> {
+  private started = false;
+  private future = new Future<ResultT>();
+
+  #logger = log();
+
+  get done(): boolean {
+    return this.future.done;
+  }
+
+  complete(result: ResultT | Error): void {
+    if (this.future.done) {
+      throw new Error(`${this.constructor.name} is already done`);
+    }
+
+    if (result instanceof Error) {
+      this.future.reject(result);
+    } else {
+      this.future.resolve(result);
+    }
+
+    const speechHandle = speechHandleStorage.getStore();
+    if (speechHandle) {
+      speechHandle._maybeRunFinalOutput = result;
+    }
+  }
+
+  async run(): Promise<ResultT> {
+    if (this.started) {
+      throw new Error(
+        `Task ${this.constructor.name} has already started and cannot be awaited multiple times`,
+      );
+    }
+    this.started = true;
+
+    const currentTask = Task.current();
+    if (!currentTask) {
+      throw new Error(`${this.constructor.name} must be executed inside a Task context`);
+    }
+
+    const taskInfo = _getActivityTaskInfo(currentTask);
+    if (!taskInfo || !taskInfo.inlineTask) {
+      throw new Error(
+        `${this.constructor.name} should only be awaited inside function tools or the onEnter/onExit methods of an Agent`,
+      );
+    }
+
+    const speechHandle = speechHandleStorage.getStore();
+    const oldActivity = agentActivityStorage.getStore();
+    if (!oldActivity) {
+      throw new Error(`${this.constructor.name} must be executed inside an AgentActivity context`);
+    }
+
+    currentTask.addDoneCallback(() => {
+      if (this.future.done) return;
+
+      // If the Task finished before the AgentTask was completed, complete the AgentTask with an error.
+      this.#logger.error(`The Task finished before ${this.constructor.name} was completed.`);
+      this.complete(new Error(`The Task finished before ${this.constructor.name} was completed.`));
+    });
+
+    const oldAgent = oldActivity.agent;
+    const session = oldActivity.agentSession;
+
+    const blockedTasks: Task<any>[] = [currentTask];
+    const onEnterTask = oldActivity._onEnterTask;
+
+    if (onEnterTask && !onEnterTask.done && onEnterTask !== currentTask) {
+      blockedTasks.push(onEnterTask);
+    }
+
+    if (
+      taskInfo.functionCall &&
+      oldActivity.llm instanceof RealtimeModel &&
+      !oldActivity.llm.capabilities.manualFunctionCalls
+    ) {
+      this.#logger.error(
+        `Realtime model does not support resuming function calls from chat context, ` +
+          `using AgentTask inside a function tool may have unexpected behavior.`,
+      );
+    }
+
+    await session._updateActivity(this, {
+      previousActivity: 'pause',
+      newActivity: 'start',
+      blockedTasks,
+    });
+
+    let runState = session._globalRunState;
+    if (speechHandle && runState && !runState.done()) {
+      // Only unwatch the parent speech handle if there are other handles keeping the run alive.
+      // When watchedHandleCount is 1 (only the parent), unwatching would drop it to 0 and
+      // mark the run done prematurely — before function_call_output and assistant message arrive.
+      if (runState._watchedHandleCount() > 1) {
+        runState._unwatchHandle(speechHandle);
+      }
+      // it is OK to call _markDoneIfNeeded here, the above _updateActivity will call onEnter
+      // and newly added handles keep the run alive.
+      runState._markDoneIfNeeded();
+    }
+
+    try {
+      return await this.future.await;
+    } finally {
+      // runState could have changed after future resolved
+      runState = session._globalRunState;
+
+      if (session.currentAgent !== this) {
+        this.#logger.warn(
+          `${this.constructor.name} completed, but the agent has changed in the meantime. ` +
+            `Ignoring handoff to the previous agent, likely due to AgentSession.updateAgent being invoked.`,
+        );
+        await oldActivity.close();
+      } else {
+        if (speechHandle && runState && !runState.done()) {
+          runState._watchHandle(speechHandle);
+        }
+
+        const mergedChatCtx = oldAgent._chatCtx.merge(this._chatCtx, {
+          excludeFunctionCall: true,
+          excludeInstructions: true,
+        });
+        oldAgent._chatCtx.items = mergedChatCtx.items;
+
+        await session._updateActivity(oldAgent, {
+          previousActivity: 'close',
+          newActivity: 'resume',
+          waitOnEnter: false,
+        });
+      }
+    }
+  }
+}