@livekit/agents 1.1.0-dev.0 → 1.2.0

package/src/voice/agent.test.ts

@@ -7,7 +7,11 @@ import { tool } from '../llm/index.js';
 import { initializeLogger } from '../log.js';
 import { Task } from '../utils.js';
 import { Agent, AgentTask, _setActivityTaskInfo } from './agent.js';
-import { agentActivityStorage } from './agent_activity.js';
+import { AgentActivity, agentActivityStorage } from './agent_activity.js';
+import { defaultEndpointingOptions } from './turn_config/endpointing.js';
+import { defaultInterruptionOptions } from './turn_config/interruption.js';
+
+vi.mock('ofetch', () => ({ ofetch: vi.fn() }));
 
 initializeLogger({ pretty: false, level: 'error' });
 
@@ -215,4 +219,207 @@ describe('Agent', () => {
     await expect(wrapper.result).resolves.toBe('ok');
     expect(closeOldActivity).toHaveBeenCalledTimes(1);
   });
+
+  describe('Agent constructor option migration', () => {
+    it('should set allowInterruptions to false via deprecated constructor field', () => {
+      const agent = new Agent({ instructions: 'test', allowInterruptions: false });
+      expect(agent.turnHandling?.interruption?.enabled).toBe(false);
+    });
+
+    it('should not set derived properties when no compatibility fields are provided', () => {
+      const agent = new Agent({ instructions: 'test' });
+      expect(agent.turnHandling).toBeUndefined();
+    });
+
+    it('should expose minConsecutiveSpeechDelay', () => {
+      const agent = new Agent({ instructions: 'test', minConsecutiveSpeechDelay: 1.5 });
+      expect(agent.minConsecutiveSpeechDelay).toBe(1.5);
+    });
+
+    it('should ignore deprecated constructor fields when turnHandling is provided', () => {
+      const agent = new Agent({
+        instructions: 'test',
+        turnHandling: {
+          endpointing: { minDelay: 999 },
+          interruption: {},
+          turnDetection: 'vad',
+        },
+        allowInterruptions: false,
+      });
+      expect(agent.turnHandling?.endpointing?.minDelay).toBe(999);
+      expect(agent.turnHandling?.endpointing?.maxDelay).toBeUndefined();
+      expect(agent.turnHandling?.interruption?.enabled).toBeUndefined();
+      expect(agent.turnHandling?.turnDetection).toBe('vad');
+    });
+
+    it('should let turnHandling override deprecated constructor fields on conflicts', () => {
+      const agent = new Agent({
+        instructions: 'test',
+        turnHandling: {
+          endpointing: { minDelay: 999, maxDelay: 4000 },
+          interruption: { enabled: true },
+          turnDetection: 'vad',
+        },
+        allowInterruptions: false,
+        turnDetection: 'stt',
+      });
+      expect(agent.turnHandling?.endpointing?.minDelay).toBe(999);
+      expect(agent.turnHandling?.endpointing?.maxDelay).toBe(4000);
+      expect(agent.turnHandling?.interruption?.enabled).toBe(true);
+      expect(agent.turnHandling?.turnDetection).toBe('vad');
+    });
+
+    it('should set interruptionDetection from turnHandling.interruption.mode', () => {
+      const agent = new Agent({
+        instructions: 'test',
+        turnHandling: {
+          interruption: { mode: 'adaptive' },
+          endpointing: {},
+          turnDetection: undefined,
+        },
+      });
+      expect(agent.turnHandling?.interruption?.mode).toBe('adaptive');
+    });
+
+    it('should let AgentActivity prefer agent-level overrides over session defaults', () => {
+      const agent = new Agent({
+        instructions: 'test',
+        turnHandling: {
+          endpointing: { minDelay: 111, maxDelay: 222 },
+          interruption: { enabled: false },
+          turnDetection: 'manual',
+        },
+      });
+      const session = {
+        options: {
+          turnHandling: {
+            endpointing: defaultEndpointingOptions,
+            interruption: defaultInterruptionOptions,
+          },
+        },
+        turnDetection: 'stt',
+        useTtsAlignedTranscript: true,
+        vad: undefined,
+        stt: undefined,
+        llm: undefined,
+        tts: undefined,
+        interruptionDetection: undefined,
+      } as any;
+
+      const activity = new AgentActivity(agent as any, session);
+
+      expect(activity.allowInterruptions).toBe(false);
+      expect(activity.turnDetection).toBe('manual');
+      expect(activity.turnHandling.endpointing?.minDelay).toBe(111);
+      expect(activity.turnHandling.endpointing?.maxDelay).toBe(222);
+    });
+
+    it('should disable adaptive interruption detection in default mode when prerequisites are missing', () => {
+      const previousRemoteEotUrl = process.env.LIVEKIT_REMOTE_EOT_URL;
+      process.env.LIVEKIT_REMOTE_EOT_URL = 'http://localhost:9999';
+
+      try {
+        const agent = new Agent({ instructions: 'test' });
+        const session = {
+          options: {
+            turnHandling: {
+              endpointing: defaultEndpointingOptions,
+              interruption: defaultInterruptionOptions,
+            },
+          },
+          sessionOptions: {
+            turnHandling: {
+              endpointing: defaultEndpointingOptions,
+              interruption: defaultInterruptionOptions,
+            },
+          },
+          turnDetection: 'manual',
+          useTtsAlignedTranscript: true,
+          vad: {},
+          stt: {
+            capabilities: {
+              alignedTranscript: true,
+              streaming: true,
+            },
+          },
+          llm: undefined,
+          tts: undefined,
+          interruptionDetection: undefined,
+        } as any;
+
+        const activity = new AgentActivity(agent as any, session);
+        expect((activity as any).interruptionDetector).toBeUndefined();
+      } finally {
+        if (previousRemoteEotUrl === undefined) {
+          delete process.env.LIVEKIT_REMOTE_EOT_URL;
+        } else {
+          process.env.LIVEKIT_REMOTE_EOT_URL = previousRemoteEotUrl;
+        }
+      }
+    });
+
+    it('should warn when session explicitly requests adaptive detection even if agent overrides it', () => {
+      const activity = Object.create(AgentActivity.prototype) as any;
+      activity.agent = {
+        turnHandling: { interruption: { mode: 'vad' } },
+        turnDetection: undefined,
+      };
+      activity.agentSession = {
+        interruptionDetection: 'adaptive',
+        turnDetection: 'manual',
+      };
+      activity.logger = { warn: vi.fn() };
+
+      expect(activity.resolveInterruptionDetector()).toBeUndefined();
+      expect(activity.logger.warn).toHaveBeenCalledWith(
+        "interruptionDetection is provided, but it's not compatible with the current configuration and will be disabled",
+      );
+    });
+
+    it('should disable adaptive interruption detection when interruptions are disabled', () => {
+      const previousRemoteEotUrl = process.env.LIVEKIT_REMOTE_EOT_URL;
+      process.env.LIVEKIT_REMOTE_EOT_URL = 'http://localhost:9999';
+
+      try {
+        const activity = Object.create(AgentActivity.prototype) as any;
+        activity.agent = {
+          turnHandling: {
+            interruption: { enabled: false },
+          },
+          turnDetection: undefined,
+          stt: undefined,
+          vad: undefined,
+          llm: undefined,
+        };
+        activity.agentSession = {
+          interruptionDetection: undefined,
+          turnDetection: 'stt',
+          sessionOptions: {
+            turnHandling: {
+              interruption: defaultInterruptionOptions,
+              endpointing: defaultEndpointingOptions,
+            },
+          },
+          stt: {
+            capabilities: {
+              alignedTranscript: true,
+              streaming: true,
+            },
+          },
+          vad: {},
+          llm: undefined,
+        };
+        activity.logger = { warn: vi.fn() };
+
+        expect(activity.resolveInterruptionDetector()).toBeUndefined();
+        expect(activity.logger.warn).not.toHaveBeenCalled();
+      } finally {
+        if (previousRemoteEotUrl === undefined) {
+          delete process.env.LIVEKIT_REMOTE_EOT_URL;
+        } else {
+          process.env.LIVEKIT_REMOTE_EOT_URL = previousRemoteEotUrl;
+        }
+      }
+    });
+  });
 });

package/src/voice/agent.ts

@@ -35,9 +35,8 @@ import { type AgentActivity, agentActivityStorage } from './agent_activity.js';
 import type { AgentSession, TurnDetectionMode } from './agent_session.js';
 import type { TimedString } from './io.js';
 import type { SpeechHandle } from './speech_handle.js';
-import type { InterruptionOptions } from './turn_config/interruption.js';
 import type { TurnHandlingOptions } from './turn_config/turn_handling.js';
-import { migrateLegacyOptions } from './turn_config/utils.js';
+import { migrateTurnHandling } from './turn_config/utils.js';
 
 export const functionCallStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();
 export const speechHandleStorage = new AsyncLocalStorage<SpeechHandle>();
@@ -113,16 +112,17 @@ export interface AgentOptions<UserData> {
   instructions: string;
   chatCtx?: ChatContext;
   tools?: ToolContext<UserData>;
-  /** @deprecated use turnHandling instead */
-  turnDetection?: TurnDetectionMode;
   stt?: STT | STTModelString;
   vad?: VAD;
   llm?: LLM | RealtimeModel | LLMModels;
   tts?: TTS | TTSModelString;
-  allowInterruptions?: boolean;
-  minConsecutiveSpeechDelay?: number;
   turnHandling?: TurnHandlingOptions;
+  minConsecutiveSpeechDelay?: number;
   useTtsAlignedTranscript?: boolean;
+  /** @deprecated use turnHandling.turnDetection instead */
+  turnDetection?: TurnDetectionMode;
+  /** @deprecated use turnHandling.interruption.enabled instead */
+  allowInterruptions?: boolean;
 }
 
 export class Agent<UserData = any> {
@@ -131,9 +131,9 @@ export class Agent<UserData = any> {
   private _vad?: VAD;
   private _llm?: LLM | RealtimeModel;
   private _tts?: TTS;
-  private turnHandling?: TurnHandlingOptions;
-  private _interruptionDetection: InterruptionOptions['mode'];
-  private _allowInterruptions?: boolean;
+  private _turnHandling?: Partial<TurnHandlingOptions>;
+
+  private _minConsecutiveSpeechDelay?: number;
   private _useTtsAlignedTranscript?: boolean;
 
   /** @internal */
@@ -158,14 +158,14 @@ export class Agent<UserData = any> {
     vad,
     llm,
     tts,
+    allowInterruptions,
     turnHandling,
+    minConsecutiveSpeechDelay,
     useTtsAlignedTranscript,
-    allowInterruptions,
   }: AgentOptions<UserData>) {
     if (id) {
       this._id = id;
     } else {
-      // Convert class name to snake_case
       const className = this.constructor.name;
       if (className === 'Agent') {
         this._id = 'default_agent';
@@ -185,11 +185,13 @@ export class Agent<UserData = any> {
         })
       : ChatContext.empty();
 
-    const migratedOptions = migrateLegacyOptions({
+    const resolvedTurnHandling = migrateTurnHandling({
       turnDetection,
-      options: { turnHandling, allowInterruptions },
+      allowInterruptions,
+      turnHandling,
     });
-    this.turnHandling = migratedOptions.options.turnHandling;
+    this._turnHandling =
+      Object.keys(resolvedTurnHandling).length > 0 ? resolvedTurnHandling : undefined;
 
     this._vad = vad;
 
@@ -211,10 +213,7 @@ export class Agent<UserData = any> {
       this._tts = tts;
     }
 
-    this._interruptionDetection = this.turnHandling?.interruption.mode;
-    if (this.turnHandling?.interruption.mode !== undefined) {
-      this._allowInterruptions = !!this.turnHandling.interruption.mode;
-    }
+    this._minConsecutiveSpeechDelay = minConsecutiveSpeechDelay;
     this._useTtsAlignedTranscript = useTtsAlignedTranscript;
 
     this._agentActivity = undefined;
@@ -260,12 +259,12 @@ export class Agent<UserData = any> {
     return this.getActivityOrThrow().agentSession as AgentSession<UserData>;
   }
 
-  get interruptionDetection(): InterruptionOptions['mode'] {
-    return this._interruptionDetection;
+  get turnHandling(): Partial<TurnHandlingOptions> | undefined {
+    return this._turnHandling;
   }
 
-  get allowInterruptions(): boolean | undefined {
-    return this._allowInterruptions;
+  get minConsecutiveSpeechDelay(): number | undefined {
+    return this._minConsecutiveSpeechDelay;
   }
 
   async onEnter(): Promise<void> {}

package/src/voice/agent_activity.test.ts

@@ -0,0 +1,194 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Regression tests for mainTask speech handle processing.
+ *
+ * When a speech handle is interrupted after _authorizeGeneration() but before the
+ * reply task calls _markGenerationDone(), mainTask hangs on _waitForGeneration()
+ * indefinitely. All subsequent speech handles queue behind it and the agent becomes
+ * unresponsive.
+ *
+ * Fix: race _waitForGeneration() against the interrupt future via waitIfNotInterrupted().
+ *
+ * Related: #1124, #1089, #836
+ */
+import { Heap } from 'heap-js';
+import { describe, expect, it, vi } from 'vitest';
+import { Future } from '../utils.js';
+import { AgentActivity } from './agent_activity.js';
+import { SpeechHandle } from './speech_handle.js';
+
+// Break circular dependency: agent_activity.ts → agent.js → beta/workflows/task_group.ts
+vi.mock('./agent.js', () => {
+  class Agent {}
+  class AgentTask extends Agent {}
+  class StopResponse {}
+  return {
+    Agent,
+    AgentTask,
+    StopResponse,
+    _getActivityTaskInfo: () => null,
+    _setActivityTaskInfo: () => {},
+    functionCallStorage: {
+      getStore: () => undefined,
+      enterWith: () => {},
+      run: (_: unknown, fn: () => unknown) => fn(),
+    },
+    speechHandleStorage: {
+      getStore: () => undefined,
+      enterWith: () => {},
+    },
+  };
+});
+
+vi.mock('../version.js', () => ({ version: '0.0.0-test' }));
+
+async function raceTimeout(promise: Promise<unknown>, ms: number): Promise<'resolved' | 'timeout'> {
+  let timer: ReturnType<typeof setTimeout>;
+  const timeout = new Promise<'timeout'>((resolve) => {
+    timer = setTimeout(() => resolve('timeout'), ms);
+  });
+  return Promise.race([promise.then(() => 'resolved' as const), timeout]).finally(() =>
+    clearTimeout(timer),
+  );
+}
+
+/**
+ * Build a minimal stand-in with just enough state for mainTask to run.
+ *
+ * mainTask accesses: q_updated, speechQueue, _currentSpeech, _schedulingPaused,
+ * getDrainPendingSpeechTasks(), and logger. We provide stubs for all of these,
+ * then bind the real AgentActivity.prototype.mainTask to this object.
+ */
+function buildMainTaskRunner() {
+  const q_updated = new Future<void>();
+  type HeapItem = [number, number, SpeechHandle];
+  const speechQueue = new Heap<HeapItem>((a: HeapItem, b: HeapItem) => b[0] - a[0] || a[1] - b[1]);
+
+  const fakeActivity = {
+    q_updated,
+    speechQueue,
+    _currentSpeech: undefined as SpeechHandle | undefined,
+    _schedulingPaused: false,
+    getDrainPendingSpeechTasks: () => [],
+    logger: {
+      info: () => {},
+      debug: () => {},
+      warn: () => {},
+      error: () => {},
+    },
+  };
+
+  const mainTask = (AgentActivity.prototype as Record<string, unknown>).mainTask as (
+    signal: AbortSignal,
+  ) => Promise<void>;
+
+  return {
+    fakeActivity,
+    mainTask: mainTask.bind(fakeActivity),
+    speechQueue,
+    q_updated,
+  };
+}
+
+describe('AgentActivity - mainTask', () => {
+  it('should recover when speech handle is interrupted after authorization', async () => {
+    const { fakeActivity, mainTask, speechQueue, q_updated } = buildMainTaskRunner();
+
+    const handle = SpeechHandle.create({ allowInterruptions: true });
+
+    speechQueue.push([SpeechHandle.SPEECH_PRIORITY_NORMAL, 1, handle]);
+    handle._markScheduled();
+    q_updated.resolve();
+
+    const ac = new AbortController();
+    const mainTaskPromise = mainTask(ac.signal);
+
+    // Give mainTask time to pop the handle and call _authorizeGeneration
+    await new Promise((r) => setTimeout(r, 50));
+
+    // Interrupt while waiting for generation
+    handle.interrupt();
+
+    // Let mainTask react to the interrupt, then signal exit
+    await new Promise((r) => setTimeout(r, 50));
+    fakeActivity._schedulingPaused = true;
+    fakeActivity.q_updated = new Future();
+    fakeActivity.q_updated.resolve();
+    ac.abort();
+
+    const result = await raceTimeout(mainTaskPromise, 2000);
+    expect(result).toBe('resolved');
+  });
+
+  it('should process next queued handle after an interrupted one', async () => {
+    const { fakeActivity, mainTask, speechQueue, q_updated } = buildMainTaskRunner();
+
+    const handleA = SpeechHandle.create({ allowInterruptions: true });
+    const handleB = SpeechHandle.create({ allowInterruptions: true });
+
+    speechQueue.push([SpeechHandle.SPEECH_PRIORITY_NORMAL, 1, handleA]);
+    handleA._markScheduled();
+    speechQueue.push([SpeechHandle.SPEECH_PRIORITY_NORMAL, 2, handleB]);
+    handleB._markScheduled();
+    q_updated.resolve();
+
+    const ac = new AbortController();
+    const mainTaskPromise = mainTask(ac.signal);
+
+    // Wait for mainTask to pick up handle A
+    await new Promise((r) => setTimeout(r, 50));
+
+    // Interrupt handle A
+    handleA.interrupt();
+
+    // Wait for mainTask to move to handle B and authorize it
+    await new Promise((r) => setTimeout(r, 50));
+
+    // Resolve handle B's generation (simulating normal reply task completion).
+    // If mainTask is stuck on handle A (bug), handle B was never authorized and this
+    // throws — we catch it and let the timeout assert the real failure.
+    try {
+      handleB._markGenerationDone();
+    } catch {
+      // Expected when fix is absent: handle B has no active generation
+    }
+
+    // Let mainTask finish
+    await new Promise((r) => setTimeout(r, 50));
+    fakeActivity._schedulingPaused = true;
+    fakeActivity.q_updated = new Future();
+    fakeActivity.q_updated.resolve();
+    ac.abort();
+
+    const result = await raceTimeout(mainTaskPromise, 2000);
+    expect(result).toBe('resolved');
+  });
+
+  it('should skip handles that were interrupted before being popped', async () => {
+    const { fakeActivity, mainTask, speechQueue, q_updated } = buildMainTaskRunner();
+
+    const handle = SpeechHandle.create({ allowInterruptions: true });
+
+    // Interrupt before mainTask ever sees it
+    handle.interrupt();
+
+    speechQueue.push([SpeechHandle.SPEECH_PRIORITY_NORMAL, 1, handle]);
+    handle._markScheduled();
+    q_updated.resolve();
+
+    const ac = new AbortController();
+    const mainTaskPromise = mainTask(ac.signal);
+
+    await new Promise((r) => setTimeout(r, 50));
+    fakeActivity._schedulingPaused = true;
+    fakeActivity.q_updated = new Future();
+    fakeActivity.q_updated.resolve();
+    ac.abort();
+
+    const result = await raceTimeout(mainTaskPromise, 2000);
+    expect(result).toBe('resolved');
+  });
+});