@livekit/agents 1.0.35 → 1.0.36-dev.0

package/src/utils/ws_transport.test.ts

@@ -0,0 +1,282 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { WebSocket, WebSocketServer } from 'ws';
+import { webSocketStream } from './ws_transport.js';
+
+describe('webSocketStream', () => {
+  describe('readable stream', () => {
+    it('receives messages from the WebSocket', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+
+      wss.on('connection', (serverWs) => {
+        serverWs.send('hello');
+        serverWs.send('world');
+        serverWs.close();
+      });
+
+      const { readable } = webSocketStream(`ws://localhost:${port}`);
+      const reader = readable.getReader();
+
+      const messages: string[] = [];
+      try {
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          messages.push(Buffer.from(value).toString());
+        }
+      } finally {
+        reader.releaseLock();
+      }
+
+      expect(messages).toEqual(['hello', 'world']);
+
+      wss.close();
+    });
+
+    it('handles binary messages', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+
+      const binaryData = new Uint8Array([1, 2, 3, 4, 5]);
+
+      wss.on('connection', (serverWs) => {
+        serverWs.send(binaryData);
+        serverWs.close();
+      });
+
+      const { readable } = webSocketStream(`ws://localhost:${port}`);
+      const reader = readable.getReader();
+
+      const chunks: Uint8Array[] = [];
+      try {
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          chunks.push(new Uint8Array(value));
+        }
+      } finally {
+        reader.releaseLock();
+      }
+
+      expect(chunks).toHaveLength(1);
+      expect(Array.from(chunks[0]!)).toEqual([1, 2, 3, 4, 5]);
+
+      wss.close();
+    });
+
+    it('handles empty stream when connection closes immediately', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+
+      wss.on('connection', (serverWs) => {
+        serverWs.close();
+      });
+      const { readable } = webSocketStream(`ws://localhost:${port}`);
+      const reader = readable.getReader();
+
+      const chunks: Uint8Array[] = [];
+      try {
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          chunks.push(value);
+        }
+      } finally {
+        reader.releaseLock();
+      }
+
+      expect(chunks).toEqual([]);
+
+      wss.close();
+    });
+  });
+
+  describe('writable stream', () => {
+    it('sends messages through the WebSocket', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+      const ws = new WebSocket(`ws://localhost:${port}`);
+
+      const connected = new Promise<void>((resolve) => {
+        ws.on('open', resolve);
+      });
+
+      const messagesReceived: string[] = [];
+      const serverClosed = new Promise<void>((resolve) => {
+        wss.on('connection', (serverWs) => {
+          serverWs.on('message', (data) => {
+            messagesReceived.push(data.toString());
+          });
+          serverWs.on('close', resolve);
+        });
+      });
+
+      await connected;
+      const { writable } = webSocketStream(`ws://localhost:${port}`);
+      const writer = writable.getWriter();
+
+      await writer.write(new TextEncoder().encode('hello'));
+      await writer.write(new TextEncoder().encode('world'));
+      await writer.close();
+
+      await serverClosed;
+
+      expect(messagesReceived).toEqual(['hello', 'world']);
+
+      wss.close();
+    });
+
+    it('sends binary data through the WebSocket', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+
+      const chunksReceived: Buffer[] = [];
+      const serverClosed = new Promise<void>((resolve) => {
+        wss.on('connection', (serverWs) => {
+          serverWs.on('message', (data) => {
+            chunksReceived.push(Buffer.from(data as Buffer));
+          });
+          serverWs.on('close', resolve);
+        });
+      });
+
+      const { writable } = webSocketStream(`ws://localhost:${port}`);
+      const writer = writable.getWriter();
+
+      const binaryData = new Uint8Array([10, 20, 30, 40, 50]);
+      await writer.write(binaryData);
+      await writer.close();
+
+      await serverClosed;
+
+      expect(chunksReceived).toHaveLength(1);
+      expect(Array.from(chunksReceived[0]!)).toEqual([10, 20, 30, 40, 50]);
+
+      wss.close();
+    });
+
+    it('buffers writes if readyState is CONNECTING', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+
+      const { writable } = webSocketStream(`ws://localhost:${port}`);
+      const writer = writable.getWriter();
+
+      const messagesReceived: string[] = [];
+      const serverClosed = new Promise<void>((resolve) => {
+        wss.on('connection', (serverWs) => {
+          serverWs.on('message', (data) => {
+            messagesReceived.push(data.toString());
+          });
+          serverWs.on('close', resolve);
+        });
+      });
+
+      // These writes should be buffered
+      await writer.write(new TextEncoder().encode('buffered message'));
+      await writer.close();
+
+      await serverClosed;
+
+      expect(messagesReceived).toEqual(['buffered message']);
+
+      wss.close();
+    });
+  });
+
+  describe('bidirectional communication', () => {
+    it('supports echo pattern with readable and writable', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+
+      // Server echoes messages back
+      wss.on('connection', (serverWs) => {
+        serverWs.on('message', (data) => {
+          serverWs.send(data);
+        });
+      });
+
+      const { readable, writable } = webSocketStream(`ws://localhost:${port}`);
+      const writer = writable.getWriter();
+      const reader = readable.getReader();
+
+      // Send messages
+      await writer.write(new TextEncoder().encode('ping1'));
+      await writer.write(new TextEncoder().encode('ping2'));
+
+      // Read echoed responses
+      const { value: response1 } = await reader.read();
+      const { value: response2 } = await reader.read();
+
+      expect(Buffer.from(response1!).toString()).toBe('ping1');
+      expect(Buffer.from(response2!).toString()).toBe('ping2');
+
+      reader.releaseLock();
+      await writer.close();
+
+      wss.close();
+    });
+  });
+
+  describe('error handling', () => {
+    it('readable stream ends when WebSocket closes unexpectedly', async () => {
+      const wss = await new Promise<WebSocketServer>((resolve) => {
+        const server: WebSocketServer = new WebSocketServer({ port: 0 }, () => resolve(server));
+      });
+
+      const port = (wss.address() as { port: number }).port;
+
+      wss.on('connection', (serverWs) => {
+        serverWs.send('before close');
+        // Terminate connection abruptly
+        serverWs.terminate();
+      });
+
+      const { readable } = webSocketStream(`ws://localhost:${port}`);
+      const reader = readable.getReader();
+
+      const chunks: string[] = [];
+      try {
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          chunks.push(Buffer.from(value).toString());
+        }
+      } catch (error) {
+        console.error(error);
+        // Connection terminated, stream may error
+      } finally {
+        reader.releaseLock();
+      }
+
+      // Should have received the message sent before termination
+      expect(chunks).toContain('before close');
+
+      wss.close();
+    });
+  });
+});

package/src/utils/ws_transport.ts

@@ -0,0 +1,22 @@
+import { Readable, Writable } from 'node:stream';
+import WebSocket, { createWebSocketStream } from 'ws';
+
+export function webSocketStream(wsUrl: string) {
+  const ws = new WebSocket(wsUrl);
+  const duplex = createWebSocketStream(ws);
+  duplex.on('error', console.error);
+
+  // End the write side when the read side ends to properly close the stream.
+  // This is needed because Readable.toWeb() waits for both sides of the duplex
+  // to close before signaling done on the ReadableStream.
+  duplex.on('end', () => {
+    duplex.end();
+  });
+
+  // Convert the writable side
+  const writable = Writable.toWeb(duplex);
+  // Convert the readable side
+  const readable = Readable.toWeb(duplex);
+
+  return { readable, writable, close: ws.close };
+}

package/src/voice/agent_activity.ts

@@ -41,6 +41,8 @@ import { recordRealtimeMetrics, traceTypes, tracer } from '../telemetry/index.js
 import { splitWords } from '../tokenize/basic/word.js';
 import { TTS, type TTSError } from '../tts/tts.js';
 import { Future, Task, cancelAndWait, waitFor } from '../utils.js';
+import type { InterruptionEvent } from '../inference/interruption/interruption.js';
+import { InterruptionEventType } from '../inference/interruption/interruption.js';
 import { VAD, type VADEvent } from '../vad.js';
 import type { Agent, ModelSettings } from './agent.js';
 import { StopResponse, asyncLocalStorage } from './agent.js';
@@ -112,6 +114,24 @@ export class AgentActivity implements RecognitionHooks {
   _mainTask?: Task<void>;
   _userTurnCompletedTask?: Promise<void>;
 
+  /**
+   * Notify that agent started speaking.
+   * This enables interruption detection in AudioRecognition.
+   * @internal
+   */
+  notifyAgentSpeechStarted(): void {
+    this.audioRecognition?.onStartOfAgentSpeech();
+  }
+
+  /**
+   * Notify that agent stopped speaking.
+   * This disables interruption detection in AudioRecognition.
+   * @internal
+   */
+  notifyAgentSpeechEnded(): void {
+    this.audioRecognition?.onEndOfAgentSpeech();
+  }
+
   constructor(agent: Agent, agentSession: AgentSession) {
     this.agent = agent;
     this.agentSession = agentSession;
@@ -292,6 +312,7 @@ export class AgentActivity implements RecognitionHooks {
         // Disable stt node if stt is not provided
         stt: this.stt ? (...args) => this.agent.sttNode(...args) : undefined,
         vad: this.vad,
+        interruptionDetector: this.agentSession.interruptionDetector,
         turnDetector: typeof this.turnDetection === 'string' ? undefined : this.turnDetection,
         turnDetectionMode: this.turnDetectionMode,
         minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
@@ -697,6 +718,46 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
+  onInterruption(ev: InterruptionEvent): void {
+    if (ev.type !== InterruptionEventType.INTERRUPTION) {
+      // Only handle actual interruptions, not overlap_speech_ended events
+      return;
+    }
+
+    this.logger.info(
+      {
+        probability: ev.probability,
+        detectionDelay: ev.detectionDelay,
+        totalDuration: ev.totalDuration,
+      },
+      'adaptive interruption detected',
+    );
+
+    // Similar to onVADInferenceDone but triggered by the adaptive interruption detector
+    if (this.turnDetection === 'manual' || this.turnDetection === 'realtime_llm') {
+      return;
+    }
+
+    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
+      return;
+    }
+
+    this.realtimeSession?.startUserActivity();
+
+    if (
+      this._currentSpeech &&
+      !this._currentSpeech.interrupted &&
+      this._currentSpeech.allowInterruptions
+    ) {
+      this.logger.info(
+        { 'speech id': this._currentSpeech.id },
+        'speech interrupted by adaptive interruption detector',
+      );
+      this.realtimeSession?.interrupt();
+      this._currentSpeech.interrupt();
+    }
+  }
+
   onInterimTranscript(ev: SpeechEvent): void {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.userTranscription) {
       // skip stt transcription if userTranscription is enabled on the realtime model

package/src/voice/agent_session.ts

@@ -15,6 +15,7 @@ import {
   type STTModelString,
   type TTSModelString,
 } from '../inference/index.js';
+import type { AdaptiveInterruptionDetector } from '../inference/interruption/AdaptiveInterruptionDetector.js';
 import { type JobContext, getJobContext } from '../job.js';
 import type { FunctionCall, FunctionCallOutput } from '../llm/chat_context.js';
 import { AgentHandoffItem, ChatContext, ChatMessage } from '../llm/chat_context.js';
@@ -106,6 +107,7 @@ export type AgentSessionOptions<UserData = UnknownUserData> = {
   vad?: VAD;
   llm?: LLM | RealtimeModel | LLMModels;
   tts?: TTS | TTSModelString;
+  interruptionDetector?: AdaptiveInterruptionDetector;
   userData?: UserData;
   voiceOptions?: Partial<VoiceOptions>;
   connOptions?: SessionConnectOptions;
@@ -167,6 +169,8 @@ export class AgentSession<
   /** @internal - Timestamp when the session started (milliseconds) */
   _startedAt?: number;
 
+  interruptionDetector?: AdaptiveInterruptionDetector;
+
   constructor(opts: AgentSessionOptions<UserData>) {
     super();
 
@@ -176,6 +180,7 @@ export class AgentSession<
       llm,
       tts,
       turnDetection,
+      interruptionDetector,
       userData,
       voiceOptions = defaultVoiceOptions,
       connOptions,
@@ -212,6 +217,7 @@ export class AgentSession<
     }
 
     this.turnDetection = turnDetection;
+    this.interruptionDetector = interruptionDetector;
     this._userData = userData;
 
     // configurable IO
@@ -637,6 +643,8 @@ export class AgentSession<
       return;
     }
 
+    const oldState = this._agentState;
+
     if (state === 'speaking') {
       // Reset error counts when agent starts speaking
       this.llmErrorCounts = 0;
@@ -651,13 +659,25 @@ export class AgentSession<
         // TODO(brian): PR4 - Set participant attributes if roomIO.room.localParticipant is available
         // (Ref: Python agent_session.py line 1161-1164)
       }
+
+      // Notify AudioRecognition that agent started speaking (for interruption detection)
+      this.activity?.notifyAgentSpeechStarted();
+    } else if (oldState === 'speaking') {
+      // Agent stopped speaking
+      if (this.agentSpeakingSpan !== undefined) {
+        // TODO(brian): PR4 - Set ATTR_END_TIME attribute if available
+        this.agentSpeakingSpan.end();
+        this.agentSpeakingSpan = undefined;
+      }
+
+      // Notify AudioRecognition that agent stopped speaking (for interruption detection)
+      this.activity?.notifyAgentSpeechEnded();
     } else if (this.agentSpeakingSpan !== undefined) {
-      // TODO(brian): PR4 - Set ATTR_END_TIME attribute if available
+      // Non-speaking to non-speaking transition but span is still open
       this.agentSpeakingSpan.end();
       this.agentSpeakingSpan = undefined;
     }
 
-    const oldState = this._agentState;
     this._agentState = state;
 
     // Handle user away timer based on state changes