@livekit/agents 1.0.48 → 1.1.0-dev.0

package/dist/voice/audio_recognition.d.cts

@@ -1,8 +1,10 @@
 /// <reference types="node" resolution-mode="require"/>
 import type { ParticipantKind } from '@livekit/rtc-node';
 import { AudioFrame } from '@livekit/rtc-node';
-import { type Context } from '@opentelemetry/api';
+import { type Context, type Span } from '@opentelemetry/api';
 import { ReadableStream } from 'node:stream/web';
+import type { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js';
+import { type OverlappingSpeechEvent } from '../inference/interruption/types.js';
 import { type ChatContext } from '../llm/chat_context.js';
 import { type SpeechEvent } from '../stt/stt.js';
 import { type VAD, type VADEvent } from '../vad.js';
@@ -27,6 +29,7 @@ export interface PreemptiveGenerationInfo {
     transcriptConfidence: number;
 }
 export interface RecognitionHooks {
+    onInterruption: (ev: OverlappingSpeechEvent) => void;
     onStartOfSpeech: (ev: VADEvent) => void;
     onVADInferenceDone: (ev: VADEvent) => void;
     onEndOfSpeech: (ev: VADEvent) => void;
@@ -37,9 +40,13 @@ export interface RecognitionHooks {
     retrieveChatCtx: () => ChatContext;
 }
 export interface _TurnDetector {
+    /** The model name used by this turn detector. */
+    readonly model: string;
+    /** The provider name for this turn detector. */
+    readonly provider: string;
     unlikelyThreshold: (language?: string) => Promise<number | undefined>;
     supportsLanguage: (language?: string) => Promise<boolean>;
-    predictEndOfTurn(chatCtx: ChatContext): Promise<number>;
+    predictEndOfTurn(chatCtx: ChatContext, timeout?: number): Promise<number>;
 }
 export interface AudioRecognitionOptions {
     /** Hooks for recognition events. */
@@ -51,7 +58,8 @@ export interface AudioRecognitionOptions {
     /** Turn detector for end-of-turn prediction. */
     turnDetector?: _TurnDetector;
     /** Turn detection mode. */
-    turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
+    turnDetectionMode?: TurnDetectionMode;
+    interruptionDetection?: AdaptiveInterruptionDetector;
     /** Minimum endpointing delay in milliseconds. */
     minEndpointingDelay: number;
     /** Maximum endpointing delay in milliseconds. */
@@ -75,6 +83,7 @@ export interface ParticipantLike {
     kind: ParticipantKind;
 }
 export declare class AudioRecognition {
+    #private;
     private hooks;
     private stt?;
     private vad?;
@@ -108,18 +117,48 @@ export declare class AudioRecognition {
     private commitUserTurnTask?;
     private vadTask?;
     private sttTask?;
+    private interruptionTask?;
+    private interruptionDetection?;
+    private _inputStartedAt?;
+    private ignoreUserTranscriptUntil?;
+    private transcriptBuffer;
+    private isInterruptionEnabled;
+    private isAgentSpeaking;
+    private interruptionStreamChannel?;
     constructor(opts: AudioRecognitionOptions);
     /**
      * Current transcript of the user's speech, including interim transcript if available.
      */
     get currentTranscript(): string;
+    /** @internal */
+    get inputStartedAt(): number | undefined;
+    /** @internal */
+    updateOptions(options: {
+        turnDetection: TurnDetectionMode | undefined;
+    }): void;
     start(): Promise<void>;
+    stop(): Promise<void>;
+    onStartOfAgentSpeech(): Promise<boolean>;
+    onEndOfAgentSpeech(ignoreUserTranscriptUntil: number): Promise<void>;
+    /** Start interruption inference when agent is speaking and overlap speech starts. */
+    onStartOfOverlapSpeech(speechDuration: number, startedAt: number, userSpeakingSpan?: Span): Promise<void>;
+    /** End interruption inference when overlap speech ends. */
+    onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span): Promise<boolean | undefined>;
+    /**
+     * Flush held transcripts whose *end time* is after the ignoreUserTranscriptUntil timestamp.
+     * If the event has no timestamps, we assume it is the same as the next valid event.
+     */
+    private flushHeldTranscripts;
+    private shouldHoldSttEvent;
+    private trySendInterruptionSentinel;
     private ensureUserTurnSpan;
     private userTurnContext;
     private onSTTEvent;
+    private onOverlapSpeechEvent;
     private runEOUDetection;
     private createSttTask;
     private createVadTask;
+    private createInterruptionTask;
     setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void;
     detachInputAudioStream(): void;
     clearUserTurn(): void;

package/dist/voice/audio_recognition.d.ts

@@ -1,8 +1,10 @@
 /// <reference types="node" resolution-mode="require"/>
 import type { ParticipantKind } from '@livekit/rtc-node';
 import { AudioFrame } from '@livekit/rtc-node';
-import { type Context } from '@opentelemetry/api';
+import { type Context, type Span } from '@opentelemetry/api';
 import { ReadableStream } from 'node:stream/web';
+import type { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js';
+import { type OverlappingSpeechEvent } from '../inference/interruption/types.js';
 import { type ChatContext } from '../llm/chat_context.js';
 import { type SpeechEvent } from '../stt/stt.js';
 import { type VAD, type VADEvent } from '../vad.js';
@@ -27,6 +29,7 @@ export interface PreemptiveGenerationInfo {
     transcriptConfidence: number;
 }
 export interface RecognitionHooks {
+    onInterruption: (ev: OverlappingSpeechEvent) => void;
     onStartOfSpeech: (ev: VADEvent) => void;
     onVADInferenceDone: (ev: VADEvent) => void;
     onEndOfSpeech: (ev: VADEvent) => void;
@@ -37,9 +40,13 @@ export interface RecognitionHooks {
     retrieveChatCtx: () => ChatContext;
 }
 export interface _TurnDetector {
+    /** The model name used by this turn detector. */
+    readonly model: string;
+    /** The provider name for this turn detector. */
+    readonly provider: string;
     unlikelyThreshold: (language?: string) => Promise<number | undefined>;
     supportsLanguage: (language?: string) => Promise<boolean>;
-    predictEndOfTurn(chatCtx: ChatContext): Promise<number>;
+    predictEndOfTurn(chatCtx: ChatContext, timeout?: number): Promise<number>;
 }
 export interface AudioRecognitionOptions {
     /** Hooks for recognition events. */
@@ -51,7 +58,8 @@ export interface AudioRecognitionOptions {
     /** Turn detector for end-of-turn prediction. */
     turnDetector?: _TurnDetector;
     /** Turn detection mode. */
-    turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
+    turnDetectionMode?: TurnDetectionMode;
+    interruptionDetection?: AdaptiveInterruptionDetector;
     /** Minimum endpointing delay in milliseconds. */
     minEndpointingDelay: number;
     /** Maximum endpointing delay in milliseconds. */
@@ -75,6 +83,7 @@ export interface ParticipantLike {
     kind: ParticipantKind;
 }
 export declare class AudioRecognition {
+    #private;
     private hooks;
     private stt?;
     private vad?;
@@ -108,18 +117,48 @@ export declare class AudioRecognition {
     private commitUserTurnTask?;
     private vadTask?;
     private sttTask?;
+    private interruptionTask?;
+    private interruptionDetection?;
+    private _inputStartedAt?;
+    private ignoreUserTranscriptUntil?;
+    private transcriptBuffer;
+    private isInterruptionEnabled;
+    private isAgentSpeaking;
+    private interruptionStreamChannel?;
     constructor(opts: AudioRecognitionOptions);
     /**
      * Current transcript of the user's speech, including interim transcript if available.
      */
     get currentTranscript(): string;
+    /** @internal */
+    get inputStartedAt(): number | undefined;
+    /** @internal */
+    updateOptions(options: {
+        turnDetection: TurnDetectionMode | undefined;
+    }): void;
     start(): Promise<void>;
+    stop(): Promise<void>;
+    onStartOfAgentSpeech(): Promise<boolean>;
+    onEndOfAgentSpeech(ignoreUserTranscriptUntil: number): Promise<void>;
+    /** Start interruption inference when agent is speaking and overlap speech starts. */
+    onStartOfOverlapSpeech(speechDuration: number, startedAt: number, userSpeakingSpan?: Span): Promise<void>;
+    /** End interruption inference when overlap speech ends. */
+    onEndOfOverlapSpeech(endedAt: number, userSpeakingSpan?: Span): Promise<boolean | undefined>;
+    /**
+     * Flush held transcripts whose *end time* is after the ignoreUserTranscriptUntil timestamp.
+     * If the event has no timestamps, we assume it is the same as the next valid event.
+     */
+    private flushHeldTranscripts;
+    private shouldHoldSttEvent;
+    private trySendInterruptionSentinel;
     private ensureUserTurnSpan;
     private userTurnContext;
     private onSTTEvent;
+    private onOverlapSpeechEvent;
     private runEOUDetection;
     private createSttTask;
     private createVadTask;
+    private createInterruptionTask;
     setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void;
     detachInputAudioStream(): void;
     clearUserTurn(): void;

package/dist/voice/audio_recognition.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"audio_recognition.d.ts","sourceRoot":"","sources":["../../src/voice/audio_recognition.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,KAAK,OAAO,EAKb,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAK1D,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,eAAe,CAAC;AAGlE,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,QAAQ,EAAgB,MAAM,WAAW,CAAC;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGvC,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,aAAa,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qEAAqE;IACrE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,4EAA4E;IAC5E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC3C,aAAa,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACtC,mBAAmB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC/C,iBAAiB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7C,WAAW,EAAE,CAAC,IAAI,EAAE,aAAa,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,sBAAsB,EAAE,CAAC,IAAI,EAAE,wBAAwB,KAAK,IAAI,CAAC;IAEjE,eAAe,EAAE,MAAM,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC5B,iBAAiB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACtE,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,gBAAgB,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,oCAAoC;IACpC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,2BAA2B;IAC3B,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,gCAAgC;IAChC,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,gDAAgD;IAChD,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,2BAA2B;IAC3B,iBAAiB,CAAC,EAAE,OAAO,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;IAC9D,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,qCAAqC;IACrC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iCAAiC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,oBAAoB,CAAC,EAAE,MAAM,eAAe,GAAG,SAAS,CAAC;CAC1D;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,eAAe,CAAC;CACvB;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,GAAG,CAAC,CAAU;IACtB,OAAO,CAAC,GAAG,CAAC,CAAM;IAClB,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,oBAAoB,CAAC,CAAoC;IAEjE,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,sBAAsB,CAAM;IACpC,OAAO,CAAC,wBAAwB,CAAM;IACtC,OAAO,CAAC,yBAAyB,CAAgB;IACjD,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,eAAe,CAAqB;IAC5C,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAC,CAAS;IAE5B,OAAO,CAAC,YAAY,CAAC,CAAO;IAE5B,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,qBAAqB,CAAuC;IACpE,OAAO,CAAC,kBAAkB,CAA0C;IAGpE,OAAO,CAAC,aAAa,CAAC,CAAa;IACnC,OAAO,CAAC,kBAAkB,CAAC,CAAa;IACxC,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,OAAO,CAAC,CAAa;gBAEjB,IAAI,EAAE,uBAAuB;IAqBzC;;OAEG;IACH,IAAI,iBAAiB,IAAI,MAAM,CAK9B;IAEK,KAAK;IAYX,OAAO,CAAC,kBAAkB;IA0B1B,OAAO,CAAC,eAAe;YAKT,UAAU;IA6MxB,OAAO,CAAC,eAAe;YAmKT,aAAa;YAqDb,aAAa;IA0E3B,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAI3D,sBAAsB;IAItB,aAAa;IAeb,cAAc,CAAC,aAAa,EAAE,OAAO;IA8C/B,KAAK;IASX,OAAO,CAAC,gBAAgB;IAuBxB,OAAO,KAAK,oBAAoB,GAE/B;CACF"}
+{"version":3,"file":"audio_recognition.d.ts","sourceRoot":"","sources":["../../src/voice/audio_recognition.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,KAAK,OAAO,EAEZ,KAAK,IAAI,EAGV,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,oDAAoD,CAAC;AAEvG,OAAO,EAEL,KAAK,sBAAsB,EAC5B,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAM1D,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,eAAe,CAAC;AAGlE,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,QAAQ,EAAgB,MAAM,WAAW,CAAC;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGvC,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,aAAa,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qEAAqE;IACrE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,4EAA4E;IAC5E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,CAAC,EAAE,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACrD,eAAe,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACxC,kBAAkB,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC3C,aAAa,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,IAAI,CAAC;IACtC,mBAAmB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC/C,iBAAiB,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,CAAC;IAC7C,WAAW,EAAE,CAAC,IAAI,EAAE,aAAa,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,sBAAsB,EAAE,CAAC,IAAI,EAAE,wBAAwB,KAAK,IAAI,CAAC;IAEjE,eAAe,EAAE,MAAM,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC5B,iDAAiD;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,gDAAgD;IAChD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACtE,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,uBAAuB;IACtC,oCAAoC;IACpC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,2BAA2B;IAC3B,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,gCAAgC;IAChC,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,gDAAgD;IAChD,YAAY,CAAC,EAAE,aAAa,CAAC;IAC7B,2BAA2B;IAC3B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,qBAAqB,CAAC,EAAE,4BAA4B,CAAC;IACrD,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,qCAAqC;IACrC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iCAAiC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,oBAAoB,CAAC,EAAE,MAAM,eAAe,GAAG,SAAS,CAAC;CAC1D;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,eAAe,CAAC;CACvB;AAGD,qBAAa,gBAAgB;;IAC3B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,GAAG,CAAC,CAAU;IACtB,OAAO,CAAC,GAAG,CAAC,CAAM;IAClB,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,oBAAoB,CAAC,CAAoC;IAEjE,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,sBAAsB,CAAM;IACpC,OAAO,CAAC,wBAAwB,CAAM;IACtC,OAAO,CAAC,yBAAyB,CAAgB;IACjD,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,eAAe,CAAqB;IAC5C,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAC,CAAS;IAE5B,OAAO,CAAC,YAAY,CAAC,CAAO;IAE5B,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,qBAAqB,CAAuC;IACpE,OAAO,CAAC,kBAAkB,CAA0C;IAGpE,OAAO,CAAC,aAAa,CAAC,CAAa;IACnC,OAAO,CAAC,kBAAkB,CAAC,CAAa;IACxC,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,gBAAgB,CAAC,CAAa;IAGtC,OAAO,CAAC,qBAAqB,CAAC,CAA+B;IAC7D,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,yBAAyB,CAAC,CAAS;IAC3C,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,qBAAqB,CAAU;IACvC,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,yBAAyB,CAAC,CAAmD;gBAEzE,IAAI,EAAE,uBAAuB;IAyCzC;;OAEG;IACH,IAAI,iBAAiB,IAAI,MAAM,CAK9B;IAED,gBAAgB;IAChB,IAAI,cAAc,uBAEjB;IAED,gBAAgB;IAChB,aAAa,CAAC,OAAO,EAAE;QAAE,aAAa,EAAE,iBAAiB,GAAG,SAAS,CAAA;KAAE,GAAG,IAAI;IAIxE,KAAK;IAmBL,IAAI;IAMJ,oBAAoB;IAKpB,kBAAkB,CAAC,yBAAyB,EAAE,MAAM;IA4B1D,qFAAqF;IAC/E,sBAAsB,CAAC,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,IAAI;IAY/F,2DAA2D;IACrD,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,IAAI;IAWnE;;;OAGG;YACW,oBAAoB;IA4ElC,OAAO,CAAC,kBAAkB;YAkCZ,2BAA2B;IAoBzC,OAAO,CAAC,kBAAkB;IA0B1B,OAAO,CAAC,eAAe;YAKT,UAAU;IAgOxB,OAAO,CAAC,oBAAoB;IAM5B,OAAO,CAAC,eAAe;YAmKT,aAAa;YAqDb,aAAa;YA4Eb,sBAAsB;IA+EpC,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAI3D,sBAAsB;IAItB,aAAa;IAeb,cAAc,CAAC,aAAa,EAAE,OAAO;IA8C/B,KAAK;IAWX,OAAO,CAAC,gBAAgB;IAuBxB,OAAO,KAAK,oBAAoB,GAU/B;CACF"}

package/dist/voice/audio_recognition.js

@@ -5,14 +5,19 @@ import {
   trace
 } from "@opentelemetry/api";
 import { ReadableStream } from "node:stream/web";
+import { InterruptionDetectionError } from "../inference/interruption/errors.js";
+import { InterruptionStreamSentinel } from "../inference/interruption/interruption_stream.js";
+import {
+} from "../inference/interruption/types.js";
 import {} from "../llm/chat_context.js";
 import { log } from "../log.js";
 import { DeferredReadableStream, isStreamReaderReleaseError } from "../stream/deferred_stream.js";
 import { IdentityTransform } from "../stream/identity_transform.js";
 import { mergeReadableStreams } from "../stream/merge_readable_streams.js";
+import { createStreamChannel } from "../stream/stream_channel.js";
 import { SpeechEventType } from "../stt/stt.js";
 import { traceTypes, tracer } from "../telemetry/index.js";
-import { Task, delay } from "../utils.js";
+import { Task, delay, waitForAbort } from "../utils.js";
 import { VADEventType } from "../vad.js";
 import { setParticipantSpanAttributes } from "./utils.js";
 class AudioRecognition {
@@ -50,6 +55,15 @@ class AudioRecognition {
   commitUserTurnTask;
   vadTask;
   sttTask;
+  interruptionTask;
+  // interruption detection
+  interruptionDetection;
+  _inputStartedAt;
+  ignoreUserTranscriptUntil;
+  transcriptBuffer;
+  isInterruptionEnabled;
+  isAgentSpeaking;
+  interruptionStreamChannel;
   constructor(opts) {
     this.hooks = opts.recognitionHooks;
     this.stt = opts.stt;
@@ -64,9 +78,28 @@ class AudioRecognition {
     this.sttProvider = opts.sttProvider;
     this.getLinkedParticipant = opts.getLinkedParticipant;
     this.deferredInputStream = new DeferredReadableStream();
-    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
-    this.vadInputStream = vadInputStream;
-    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);
+    this.interruptionDetection = opts.interruptionDetection;
+    this.transcriptBuffer = [];
+    this.isInterruptionEnabled = !!(opts.interruptionDetection && opts.vad);
+    this.isAgentSpeaking = false;
+    if (opts.interruptionDetection) {
+      const [vadInputStream, teedInput] = this.deferredInputStream.stream.tee();
+      const [inputStream, sttInputStream] = teedInput.tee();
+      this.vadInputStream = vadInputStream;
+      this.sttInputStream = mergeReadableStreams(
+        sttInputStream,
+        this.silenceAudioTransform.readable
+      );
+      this.interruptionStreamChannel = createStreamChannel();
+      this.interruptionStreamChannel.addStreamInput(inputStream);
+    } else {
+      const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
+      this.vadInputStream = vadInputStream;
+      this.sttInputStream = mergeReadableStreams(
+        sttInputStream,
+        this.silenceAudioTransform.readable
+      );
+    }
     this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();
   }
   /**
@@ -78,6 +111,14 @@ class AudioRecognition {
     }
     return this.audioTranscript;
   }
+  /** @internal */
+  get inputStartedAt() {
+    return this._inputStartedAt;
+  }
+  /** @internal */
+  updateOptions(options) {
+    this.turnDetectionMode = options.turnDetection;
+  }
   async start() {
     this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));
     this.vadTask.result.catch((err) => {
@@ -87,6 +128,156 @@ class AudioRecognition {
     this.sttTask.result.catch((err) => {
       this.logger.error(`Error running STT task: ${err}`);
     });
+    this.interruptionTask = Task.from(
+      ({ signal }) => this.createInterruptionTask(this.interruptionDetection, signal)
+    );
+    this.interruptionTask.result.catch((err) => {
+      this.logger.error(`Error running interruption task: ${err}`);
+    });
+  }
+  async stop() {
+    var _a, _b, _c;
+    await ((_a = this.sttTask) == null ? void 0 : _a.cancelAndWait());
+    await ((_b = this.vadTask) == null ? void 0 : _b.cancelAndWait());
+    await ((_c = this.interruptionTask) == null ? void 0 : _c.cancelAndWait());
+  }
+  async onStartOfAgentSpeech() {
+    this.isAgentSpeaking = true;
+    return this.trySendInterruptionSentinel(InterruptionStreamSentinel.agentSpeechStarted());
+  }
+  async onEndOfAgentSpeech(ignoreUserTranscriptUntil) {
+    if (!this.isInterruptionEnabled) {
+      this.isAgentSpeaking = false;
+      return;
+    }
+    const inputOpen = await this.trySendInterruptionSentinel(
+      InterruptionStreamSentinel.agentSpeechEnded()
+    );
+    if (!inputOpen) {
+      this.isAgentSpeaking = false;
+      return;
+    }
+    if (this.isAgentSpeaking) {
+      if (this.ignoreUserTranscriptUntil === void 0) {
+        this.onEndOfOverlapSpeech(Date.now());
+      }
+      this.ignoreUserTranscriptUntil = this.ignoreUserTranscriptUntil ? Math.min(ignoreUserTranscriptUntil, this.ignoreUserTranscriptUntil) : ignoreUserTranscriptUntil;
+      await this.flushHeldTranscripts();
+    }
+    this.isAgentSpeaking = false;
+  }
+  /** Start interruption inference when agent is speaking and overlap speech starts. */
+  async onStartOfOverlapSpeech(speechDuration, startedAt, userSpeakingSpan) {
+    if (this.isAgentSpeaking) {
+      this.trySendInterruptionSentinel(
+        InterruptionStreamSentinel.overlapSpeechStarted(
+          speechDuration,
+          startedAt,
+          userSpeakingSpan
+        )
+      );
+    }
+  }
+  /** End interruption inference when overlap speech ends. */
+  async onEndOfOverlapSpeech(endedAt, userSpeakingSpan) {
+    if (!this.isInterruptionEnabled) {
+      return;
+    }
+    if (userSpeakingSpan && userSpeakingSpan.isRecording()) {
+      userSpeakingSpan.setAttribute(traceTypes.ATTR_IS_INTERRUPTION, "false");
+    }
+    return this.trySendInterruptionSentinel(InterruptionStreamSentinel.overlapSpeechEnded(endedAt));
+  }
+  /**
+   * Flush held transcripts whose *end time* is after the ignoreUserTranscriptUntil timestamp.
+   * If the event has no timestamps, we assume it is the same as the next valid event.
+   */
+  async flushHeldTranscripts() {
+    if (!this.isInterruptionEnabled || this.ignoreUserTranscriptUntil === void 0 || this.transcriptBuffer.length === 0) {
+      return;
+    }
+    if (!this._inputStartedAt) {
+      this.transcriptBuffer = [];
+      this.ignoreUserTranscriptUntil = void 0;
+      return;
+    }
+    let emitFromIndex = null;
+    let shouldFlush = false;
+    for (let i = 0; i < this.transcriptBuffer.length; i++) {
+      const ev = this.transcriptBuffer[i];
+      if (!ev || !ev.alternatives || ev.alternatives.length === 0) {
+        emitFromIndex = Math.min(emitFromIndex ?? i, i);
+        continue;
+      }
+      const firstAlternative = ev.alternatives[0];
+      if (firstAlternative.startTime === firstAlternative.endTime && firstAlternative.startTime === 0) {
+        this.transcriptBuffer = [];
+        this.ignoreUserTranscriptUntil = void 0;
+        return;
+      }
+      if (this.#alternativeEndsBeforeIgnoreWindow(firstAlternative)) {
+        emitFromIndex = null;
+      } else {
+        emitFromIndex = Math.min(emitFromIndex ?? i, i);
+        shouldFlush = true;
+        break;
+      }
+    }
+    const eventsToEmit = emitFromIndex !== null && shouldFlush ? this.transcriptBuffer.slice(emitFromIndex) : [];
+    this.transcriptBuffer = [];
+    this.ignoreUserTranscriptUntil = void 0;
+    for (const event of eventsToEmit) {
+      this.logger.trace(
+        {
+          event: event.type
+        },
+        "re-emitting held user transcript"
+      );
+      this.onSTTEvent(event);
+    }
+  }
+  #alternativeEndsBeforeIgnoreWindow(alternative) {
+    if (this.ignoreUserTranscriptUntil === void 0 || !this._inputStartedAt || alternative.startTime <= 0) {
+      return false;
+    }
+    return alternative.startTime * 1e3 + this._inputStartedAt < this.ignoreUserTranscriptUntil;
+  }
+  shouldHoldSttEvent(ev) {
+    if (!this.isInterruptionEnabled) {
+      return false;
+    }
+    if (this.isAgentSpeaking) {
+      return true;
+    }
+    if (ev.type === SpeechEventType.START_OF_SPEECH) {
+      this.ignoreUserTranscriptUntil = void 0;
+      this.transcriptBuffer = [];
+      return false;
+    }
+    if (this.ignoreUserTranscriptUntil === void 0) {
+      return false;
+    }
+    if (!ev.alternatives || ev.alternatives.length === 0) {
+      return true;
+    }
+    const alternative = ev.alternatives[0];
+    if (alternative.startTime !== alternative.endTime && this.#alternativeEndsBeforeIgnoreWindow(alternative)) {
+      return true;
+    }
+    return false;
+  }
+  async trySendInterruptionSentinel(frame) {
+    if (this.isInterruptionEnabled && this.interruptionStreamChannel && !this.interruptionStreamChannel.closed) {
+      try {
+        await this.interruptionStreamChannel.write(frame);
+        return true;
+      } catch (e) {
+        this.logger.warn(
+          `could not forward interruption sentinel: ${e instanceof Error ? e.message : String(e)}`
+        );
+      }
+    }
+    return false;
   }
   ensureUserTurnSpan(startTime) {
     var _a;
@@ -128,6 +319,18 @@ class AudioRecognition {
       );
       return;
     }
+    if (ev.type !== SpeechEventType.RECOGNITION_USAGE && this.isInterruptionEnabled) {
+      if (this.shouldHoldSttEvent(ev)) {
+        this.logger.trace(
+          { event: ev.type, ignoreUserTranscriptUntil: this.ignoreUserTranscriptUntil },
+          "holding STT event until ignore_user_transcript_until expires"
+        );
+        this.transcriptBuffer.push(ev);
+        return;
+      } else {
+        await this.flushHeldTranscripts();
+      }
+    }
     switch (ev.type) {
       case SpeechEventType.FINAL_TRANSCRIPT:
         const transcript = (_c = (_b = ev.alternatives) == null ? void 0 : _b[0]) == null ? void 0 : _c.text;
@@ -272,6 +475,11 @@ class AudioRecognition {
         }
     }
   }
+  onOverlapSpeechEvent(ev) {
+    if (ev.isInterruption) {
+      this.hooks.onInterruption(ev);
+    }
+  }
   runEOUDetection(chatCtx) {
     var _a;
     this.logger.debug(
@@ -467,7 +675,7 @@ class AudioRecognition {
             if (ev.rawAccumulatedSpeech > 0) {
               this.lastSpeakingTime = Date.now();
               if (this.speechStartTime === void 0) {
-                this.speechStartTime = Date.now();
+                this.speechStartTime = Date.now() - ev.rawAccumulatedSpeech;
               }
             }
             break;
@@ -492,6 +700,72 @@ class AudioRecognition {
       this.logger.debug("VAD task closed");
     }
   }
+  async createInterruptionTask(interruptionDetection, signal) {
+    if (!interruptionDetection || !this.interruptionStreamChannel) return;
+    const stream = interruptionDetection.createStream();
+    const inputReader = this.interruptionStreamChannel.stream().getReader();
+    const cleanup = async () => {
+      try {
+        signal.removeEventListener("abort", abortHandler);
+        eventReader.releaseLock();
+        await stream.close();
+      } catch (e) {
+        this.logger.debug("createInterruptionTask: error during abort handler:", e);
+      }
+    };
+    const forwardTask = (async () => {
+      try {
+        const abortPromise = waitForAbort(signal);
+        while (!signal.aborted) {
+          const res = await Promise.race([inputReader.read(), abortPromise]);
+          if (!res) break;
+          const { value, done } = res;
+          if (done) break;
+          if (value instanceof AudioFrame) {
+            const frameDurationMs = value.samplesPerChannel / value.sampleRate * 1e3;
+            this._inputStartedAt ??= Date.now() - frameDurationMs;
+          } else {
+            this._inputStartedAt ??= Date.now();
+          }
+          await stream.pushFrame(value);
+        }
+      } finally {
+        inputReader.releaseLock();
+      }
+    })();
+    const eventReader = stream.stream().getReader();
+    const abortHandler = async () => {
+      await cleanup();
+    };
+    signal.addEventListener("abort", abortHandler);
+    try {
+      const abortPromise = waitForAbort(signal);
+      while (!signal.aborted) {
+        const res = await Promise.race([eventReader.read(), abortPromise]);
+        if (!res) break;
+        const { done, value: ev } = res;
+        if (done) break;
+        this.onOverlapSpeechEvent(ev);
+      }
+    } catch (e) {
+      if (!signal.aborted) {
+        const cause = e instanceof Error ? e : new Error(String(e));
+        interruptionDetection.emitError(
+          new InterruptionDetectionError(
+            cause.message,
+            Date.now(),
+            interruptionDetection.label,
+            false
+          )
+        );
+        this.logger.error(e, "Error in interruption task");
+      }
+    } finally {
+      await cleanup();
+      await forwardTask;
+      this.logger.debug("Interruption task closed");
+    }
+  }
   setInputAudioStream(audioStream) {
     this.deferredInputStream.setSource(audioStream);
   }
@@ -546,13 +820,15 @@ class AudioRecognition {
     });
   }
   async close() {
-    var _a, _b, _c, _d;
+    var _a, _b, _c, _d, _e, _f;
     this.detachInputAudioStream();
     this.silenceAudioWriter.releaseLock();
     await ((_a = this.commitUserTurnTask) == null ? void 0 : _a.cancelAndWait());
     await ((_b = this.sttTask) == null ? void 0 : _b.cancelAndWait());
     await ((_c = this.vadTask) == null ? void 0 : _c.cancelAndWait());
     await ((_d = this.bounceEOUTask) == null ? void 0 : _d.cancelAndWait());
+    await ((_e = this.interruptionTask) == null ? void 0 : _e.cancelAndWait());
+    await ((_f = this.interruptionStreamChannel) == null ? void 0 : _f.close());
   }
   _endUserTurnSpan({
     transcript,
@@ -572,7 +848,13 @@ class AudioRecognition {
     }
   }
   get vadBaseTurnDetection() {
-    return ["vad", void 0].includes(this.turnDetectionMode);
+    if (typeof this.turnDetectionMode === "object") {
+      return false;
+    }
+    if (this.turnDetectionMode === void 0 || this.turnDetectionMode === "vad") {
+      return true;
+    }
+    return false;
   }
 }
 export {