@livekit/agents 1.0.15 → 1.0.17

package/dist/voice/agent_session.d.cts

@@ -25,6 +25,7 @@ export interface VoiceOptions {
     minEndpointingDelay: number;
     maxEndpointingDelay: number;
     maxToolSteps: number;
+    preemptiveGeneration: boolean;
 }
 export type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;
 export type AgentSessionCallbacks = {
@@ -113,7 +114,7 @@ export declare class AgentSession<UserData = UnknownUserData> extends AgentSessi
     /** @internal */
     _updateAgentState(state: AgentState): void;
     /** @internal */
-    _updateUserState(state: UserState): void;
+    _updateUserState(state: UserState, _lastSpeakingTime?: number): void;
     private onAudioInputChanged;
     private onAudioOutputChanged;
     private onTextOutputChanged;

package/dist/voice/agent_session.d.ts

@@ -25,6 +25,7 @@ export interface VoiceOptions {
     minEndpointingDelay: number;
     maxEndpointingDelay: number;
     maxToolSteps: number;
+    preemptiveGeneration: boolean;
 }
 export type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;
 export type AgentSessionCallbacks = {
@@ -113,7 +114,7 @@ export declare class AgentSession<UserData = UnknownUserData> extends AgentSessi
     /** @internal */
     _updateAgentState(state: AgentState): void;
     /** @internal */
-    _updateUserState(state: UserState): void;
+    _updateUserState(state: UserState, _lastSpeakingTime?: number): void;
     private onAudioInputChanged;
     private onAudioOutputChanged;
     private onTextOutputChanged;

package/dist/voice/agent_session.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_session.d.ts","sourceRoot":"","sources":["../../src/voice/agent_session.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EACL,sBAAsB,EACtB,KAAK,UAAU,EACf,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,UAAU,EACf,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAK3B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAU,KAAK,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC3B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,6BAA6B,EAAE,OAAO,CAAC;IACvC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,cAAc,GAAG,QAAQ,GAAG,aAAa,CAAC;AAE1F,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACvF,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACjF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACzE,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;IACzD,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,mBAAmB,CAAC,QAAQ,GAAG,eAAe,IAAI;IAC5D,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,GAAG,SAAS,CAAC;IACtC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACtC,CAAC;2CAImC,aAAa,qBAAqB,CAAC;AAFxE,qBAAa,YAAY,CACvB,QAAQ,GAAG,eAAe,CAC1B,SAAQ,iBAA+D;IACvE,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAE/B,OAAO,CAAC,KAAK,CAAC,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAA0B;IAE3C,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAA8B;IAEjD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,WAAW,CAA8B;gBAErC,IAAI,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IA6C/C,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAMvB;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAEK,KAAK,CAAC,EACV,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,aAAa,GACd,EAAE;QACD,KAAK,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,IAAI,CAAC;QACX,YAAY,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACzC,aAAa,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAC5C,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDjB,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAQ/B,cAAc;IAQd,aAAa;IAOb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAQf,SAAS;IAOT,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAC9B,GAAG,YAAY;YAsBF,cAAc;IAmB5B,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,YAAY,IAAI,KAAK,CAMxB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,gBAAgB;IAChB,UAAU,CAAC,EACT,MAAM,EACN,KAAa,EACb,KAAY,GACb,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,KAAK,CAAC,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;KACpE,GAAG,IAAI;IAOR,gBAAgB;IAChB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI;IAc1E,gBAAgB;IAChB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAK/C,gBAAgB;IAChB,iBAAiB,CAAC,KAAK,EAAE,UAAU;IAanC,gBAAgB;IAChB,gBAAgB,CAAC,KAAK,EAAE,SAAS;IAcjC,OAAO,CAAC,mBAAmB;IAU3B,OAAO,CAAC,oBAAoB;IAE5B,OAAO,CAAC,mBAAmB;YAEb,SAAS;CA6CxB"}
+{"version":3,"file":"agent_session.d.ts","sourceRoot":"","sources":["../../src/voice/agent_session.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EACL,sBAAsB,EACtB,KAAK,UAAU,EACf,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,UAAU,EACf,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAK3B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAU,KAAK,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC3B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,6BAA6B,EAAE,OAAO,CAAC;IACvC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,OAAO,CAAC;CAC/B;AAaD,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,cAAc,GAAG,QAAQ,GAAG,aAAa,CAAC;AAE1F,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACvF,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACjF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACzE,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;IACzD,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,mBAAmB,CAAC,QAAQ,GAAG,eAAe,IAAI;IAC5D,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,GAAG,SAAS,CAAC;IACtC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACtC,CAAC;2CAImC,aAAa,qBAAqB,CAAC;AAFxE,qBAAa,YAAY,CACvB,QAAQ,GAAG,eAAe,CAC1B,SAAQ,iBAA+D;IACvE,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAE/B,OAAO,CAAC,KAAK,CAAC,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAA0B;IAE3C,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAA8B;IAEjD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,WAAW,CAA8B;gBAErC,IAAI,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IA6C/C,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAMvB;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAEK,KAAK,CAAC,EACV,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,aAAa,GACd,EAAE;QACD,KAAK,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,IAAI,CAAC;QACX,YAAY,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACzC,aAAa,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAC5C,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDjB,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAQ/B,cAAc;IAQd,aAAa;IAOb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAQf,SAAS;IAOT,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAC9B,GAAG,YAAY;YAsBF,cAAc;IAmB5B,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,YAAY,IAAI,KAAK,CAMxB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,gBAAgB;IAChB,UAAU,CAAC,EACT,MAAM,EACN,KAAa,EACb,KAAY,GACb,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,KAAK,CAAC,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;KACpE,GAAG,IAAI;IAOR,gBAAgB;IAChB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI;IAc1E,gBAAgB;IAChB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAK/C,gBAAgB;IAChB,iBAAiB,CAAC,KAAK,EAAE,UAAU;IAanC,gBAAgB;IAChB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,iBAAiB,CAAC,EAAE,MAAM;IAc7D,OAAO,CAAC,mBAAmB;IAU3B,OAAO,CAAC,oBAAoB;IAE5B,OAAO,CAAC,mBAAmB;YAEb,SAAS;CA6CxB"}

package/dist/voice/agent_session.js

@@ -25,7 +25,8 @@ const defaultVoiceOptions = {
   minInterruptionWords: 0,
   minEndpointingDelay: 500,
   maxEndpointingDelay: 6e3,
-  maxToolSteps: 3
+  maxToolSteps: 3,
+  preemptiveGeneration: false
 };
 class AgentSession extends EventEmitter {
   vad;
@@ -262,7 +263,7 @@ class AgentSession extends EventEmitter {
     );
   }
   /** @internal */
-  _updateUserState(state) {
+  _updateUserState(state, _lastSpeakingTime) {
     if (this.userState === state) {
       return;
     }

package/dist/voice/agent_session.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport {\n  LLM as InferenceLLM,\n  STT as InferenceSTT,\n  TTS as InferenceTTS,\n  type LLMModels,\n  type STTModelString,\n  type TTSModelString,\n} from '../inference/index.js';\nimport { getJobContext } from '../job.js';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT | STTModelString;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel | LLMModels;\n  tts?: TTS | TTSModelString;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n\n    if (typeof stt === 'string') {\n      this.stt = InferenceSTT.fromModelString(stt);\n    } else {\n      this.stt = stt;\n    }\n\n    if (typeof llm === 'string') {\n      this.llm = InferenceLLM.fromModelString(llm);\n    } else {\n      this.llm = llm;\n    }\n\n    if (typeof tts === 'string') {\n      this.tts = InferenceTTS.fromModelString(tts);\n    } else {\n      this.tts = tts;\n    }\n\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  get history(): ChatContext {\n    return this._chatCtx;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    const tasks: Promise<void>[] = [];\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    const ctx = getJobContext();\n    if (ctx && ctx.room === room && !room.isConnected) {\n      this.logger.debug('Auto-connecting to room via job context');\n      tasks.push(ctx.connect());\n    }\n    // TODO(AJS-265): add shutdown callback to job context\n    tasks.push(this.updateActivity(this.agent));\n\n    await Promise.allSettled(tasks);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B;AAAA,EACE,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,OAIF;AACP,SAAS,qBAAqB;AAC9B,SAAS,aAAa,mBAAmB;AAGzC,SAAS,WAAW;AAMpB,SAAS,qBAAqB;AAE9B;AAAA,EACE;AAAA,EAIA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,mBAAmB;AACxC,SAAS,cAA6D;AActE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AA0BO,MAAM,qBAEF,aAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,SAAS,IAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AAEX,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,WAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,YAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,YAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAErC,UAAM,QAAyB,CAAC;AAEhC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,UAAM,MAAM,cAAc;AAC1B,QAAI,OAAO,IAAI,SAAS,QAAQ,CAAC,KAAK,aAAa;AACjD,WAAK,OAAO,MAAM,yCAAyC;AAC3D,YAAM,KAAK,IAAI,QAAQ,CAAC;AAAA,IAC1B;AAEA,UAAM,KAAK,KAAK,eAAe,KAAK,KAAK,CAAC;AAE1C,UAAM,QAAQ,WAAW,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,YAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,cAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,YAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,YAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,uBAAuB,uBAAuB,iCAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,6BAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB;AACjC,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,4BAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AAvcnB;AAwcI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,uBAAuB,OAAO,iBAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}
+{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport {\n  LLM as InferenceLLM,\n  STT as InferenceSTT,\n  TTS as InferenceTTS,\n  type LLMModels,\n  type STTModelString,\n  type TTSModelString,\n} from '../inference/index.js';\nimport { getJobContext } from '../job.js';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n  preemptiveGeneration: boolean;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n  preemptiveGeneration: false,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT | STTModelString;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel | LLMModels;\n  tts?: TTS | TTSModelString;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n\n    if (typeof stt === 'string') {\n      this.stt = InferenceSTT.fromModelString(stt);\n    } else {\n      this.stt = stt;\n    }\n\n    if (typeof llm === 'string') {\n      this.llm = InferenceLLM.fromModelString(llm);\n    } else {\n      this.llm = llm;\n    }\n\n    if (typeof tts === 'string') {\n      this.tts = InferenceTTS.fromModelString(tts);\n    } else {\n      this.tts = tts;\n    }\n\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  get history(): ChatContext {\n    return this._chatCtx;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    const tasks: Promise<void>[] = [];\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    const ctx = getJobContext();\n    if (ctx && ctx.room === room && !room.isConnected) {\n      this.logger.debug('Auto-connecting to room via job context');\n      tasks.push(ctx.connect());\n    }\n    // TODO(AJS-265): add shutdown callback to job context\n    tasks.push(this.updateActivity(this.agent));\n\n    await Promise.allSettled(tasks);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState, _lastSpeakingTime?: number) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B;AAAA,EACE,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,OAIF;AACP,SAAS,qBAAqB;AAC9B,SAAS,aAAa,mBAAmB;AAGzC,SAAS,WAAW;AAMpB,SAAS,qBAAqB;AAE9B;AAAA,EACE;AAAA,EAIA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,mBAAmB;AACxC,SAAS,cAA6D;AAetE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAAA,EACd,sBAAsB;AACxB;AA0BO,MAAM,qBAEF,aAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,SAAS,IAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AAEX,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,WAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,YAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,YAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAErC,UAAM,QAAyB,CAAC;AAEhC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,UAAM,MAAM,cAAc;AAC1B,QAAI,OAAO,IAAI,SAAS,QAAQ,CAAC,KAAK,aAAa;AACjD,WAAK,OAAO,MAAM,yCAAyC;AAC3D,YAAM,KAAK,IAAI,QAAQ,CAAC;AAAA,IAC1B;AAEA,UAAM,KAAK,KAAK,eAAe,KAAK,KAAK,CAAC;AAE1C,UAAM,QAAQ,WAAW,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,YAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,cAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,YAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,YAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,uBAAuB,uBAAuB,iCAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,6BAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB,mBAA4B;AAC7D,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,4BAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AAzcnB;AA0cI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,uBAAuB,OAAO,iBAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}

package/dist/voice/audio_recognition.cjs

@@ -45,7 +45,10 @@ class AudioRecognition {
   lastFinalTranscriptTime = 0;
   audioTranscript = "";
   audioInterimTranscript = "";
-  lastSpeakingTime = 0;
+  audioPreflightTranscript = "";
+  finalTranscriptConfidence = [];
+  lastSpeakingTime;
+  speechStartTime;
   userTurnCommitted = false;
   speaking = false;
   sampleRate;
@@ -93,7 +96,7 @@ class AudioRecognition {
     });
   }
   async onSTTEvent(ev) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r;
     if (this.turnDetectionMode === "manual" && this.userTurnCommitted && (this.bounceEOUTask === void 0 || this.bounceEOUTask.done || ev.type == import_stt.SpeechEventType.INTERIM_TRANSCRIPT)) {
       this.logger.debug(
         {
@@ -110,7 +113,8 @@ class AudioRecognition {
       case import_stt.SpeechEventType.FINAL_TRANSCRIPT:
         this.hooks.onFinalTranscript(ev);
         const transcript = (_c = (_b = ev.alternatives) == null ? void 0 : _b[0]) == null ? void 0 : _c.text;
-        this.lastLanguage = (_e = (_d = ev.alternatives) == null ? void 0 : _d[0]) == null ? void 0 : _e.language;
+        const confidence = ((_e = (_d = ev.alternatives) == null ? void 0 : _d[0]) == null ? void 0 : _e.confidence) ?? 0;
+        this.lastLanguage = (_g = (_f = ev.alternatives) == null ? void 0 : _f[0]) == null ? void 0 : _g.language;
         if (!transcript) {
           return;
         }
@@ -124,26 +128,112 @@ class AudioRecognition {
         this.lastFinalTranscriptTime = Date.now();
         this.audioTranscript += ` ${transcript}`;
         this.audioTranscript = this.audioTranscript.trimStart();
+        this.finalTranscriptConfidence.push(confidence);
+        const transcriptChanged = this.audioTranscript !== this.audioPreflightTranscript;
         this.audioInterimTranscript = "";
-        if (!this.speaking) {
-          if (!this.vad) {
-            this.lastSpeakingTime = Date.now();
+        this.audioPreflightTranscript = "";
+        if (!this.vad || this.lastSpeakingTime === void 0) {
+          this.lastSpeakingTime = Date.now();
+        }
+        if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (transcriptChanged) {
+            this.logger.debug(
+              { transcript: this.audioTranscript },
+              "triggering preemptive generation (FINAL_TRANSCRIPT)"
+            );
+            this.hooks.onPreemptiveGeneration({
+              newTranscript: this.audioTranscript,
+              transcriptConfidence: this.finalTranscriptConfidence.length > 0 ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) / this.finalTranscriptConfidence.length : 0
+            });
           }
-          if (this.vadBaseTurnDetection || this.userTurnCommitted) {
+          if (!this.speaking) {
             const chatCtx = this.hooks.retrieveChatCtx();
             this.logger.debug("running EOU detection on stt FINAL_TRANSCRIPT");
             this.runEOUDetection(chatCtx);
           }
         }
         break;
+      case import_stt.SpeechEventType.PREFLIGHT_TRANSCRIPT:
+        this.hooks.onInterimTranscript(ev);
+        const preflightTranscript = ((_i = (_h = ev.alternatives) == null ? void 0 : _h[0]) == null ? void 0 : _i.text) ?? "";
+        const preflightConfidence = ((_k = (_j = ev.alternatives) == null ? void 0 : _j[0]) == null ? void 0 : _k.confidence) ?? 0;
+        const preflightLanguage = (_m = (_l = ev.alternatives) == null ? void 0 : _l[0]) == null ? void 0 : _m.language;
+        const MIN_LANGUAGE_DETECTION_LENGTH = 5;
+        if (!this.lastLanguage || preflightLanguage && preflightTranscript.length > MIN_LANGUAGE_DETECTION_LENGTH) {
+          this.lastLanguage = preflightLanguage;
+        }
+        if (!preflightTranscript) {
+          return;
+        }
+        this.logger.debug(
+          {
+            user_transcript: preflightTranscript,
+            language: this.lastLanguage
+          },
+          "received user preflight transcript"
+        );
+        this.lastFinalTranscriptTime = Date.now();
+        this.audioPreflightTranscript = `${this.audioTranscript} ${preflightTranscript}`.trimStart();
+        this.audioInterimTranscript = preflightTranscript;
+        if (!this.vad || this.lastSpeakingTime === void 0) {
+          this.lastSpeakingTime = Date.now();
+        }
+        if (this.turnDetectionMode !== "manual" || this.userTurnCommitted) {
+          const confidenceVals = [...this.finalTranscriptConfidence, preflightConfidence];
+          this.logger.debug(
+            {
+              transcript: this.audioPreflightTranscript.length > 100 ? this.audioPreflightTranscript.slice(0, 100) + "..." : this.audioPreflightTranscript
+            },
+            "triggering preemptive generation (PREFLIGHT_TRANSCRIPT)"
+          );
+          this.hooks.onPreemptiveGeneration({
+            newTranscript: this.audioPreflightTranscript,
+            transcriptConfidence: confidenceVals.length > 0 ? confidenceVals.reduce((a, b) => a + b, 0) / confidenceVals.length : 0
+          });
+        }
+        break;
       case import_stt.SpeechEventType.INTERIM_TRANSCRIPT:
-        this.logger.debug({ transcript: (_g = (_f = ev.alternatives) == null ? void 0 : _f[0]) == null ? void 0 : _g.text }, "interim transcript");
+        this.logger.debug({ transcript: (_o = (_n = ev.alternatives) == null ? void 0 : _n[0]) == null ? void 0 : _o.text }, "interim transcript");
         this.hooks.onInterimTranscript(ev);
-        this.audioInterimTranscript = ((_i = (_h = ev.alternatives) == null ? void 0 : _h[0]) == null ? void 0 : _i.text) ?? "";
+        this.audioInterimTranscript = ((_q = (_p = ev.alternatives) == null ? void 0 : _p[0]) == null ? void 0 : _q.text) ?? "";
+        break;
+      case import_stt.SpeechEventType.START_OF_SPEECH:
+        if (this.turnDetectionMode !== "stt") break;
+        this.hooks.onStartOfSpeech({
+          type: import_vad.VADEventType.START_OF_SPEECH,
+          samplesIndex: 0,
+          timestamp: Date.now(),
+          speechDuration: 0,
+          silenceDuration: 0,
+          frames: [],
+          probability: 0,
+          inferenceDuration: 0,
+          speaking: true,
+          rawAccumulatedSilence: 0,
+          rawAccumulatedSpeech: 0
+        });
+        this.speaking = true;
+        this.lastSpeakingTime = Date.now();
+        (_r = this.bounceEOUTask) == null ? void 0 : _r.cancel();
         break;
       case import_stt.SpeechEventType.END_OF_SPEECH:
         if (this.turnDetectionMode !== "stt") break;
+        this.hooks.onEndOfSpeech({
+          type: import_vad.VADEventType.END_OF_SPEECH,
+          samplesIndex: 0,
+          timestamp: Date.now(),
+          speechDuration: 0,
+          silenceDuration: 0,
+          frames: [],
+          probability: 0,
+          inferenceDuration: 0,
+          speaking: false,
+          rawAccumulatedSilence: 0,
+          rawAccumulatedSpeech: 0
+        });
+        this.speaking = false;
         this.userTurnCommitted = true;
+        this.lastSpeakingTime = Date.now();
         if (!this.speaking) {
           const chatCtx = this.hooks.retrieveChatCtx();
           this.logger.debug("running EOU detection on stt END_OF_SPEECH");
@@ -171,7 +261,7 @@ class AudioRecognition {
       // disable EOU model if manual turn detection enabled
       this.audioTranscript && this.turnDetectionMode !== "manual" ? this.turnDetector : void 0
     );
-    const bounceEOUTask = (lastSpeakingTime) => async (controller) => {
+    const bounceEOUTask = (lastSpeakingTime, lastFinalTranscriptTime, speechStartTime) => async (controller) => {
       let endpointingDelay = this.minEndpointingDelay;
       if (turnDetector) {
         this.logger.debug("Running turn detector model");
@@ -198,21 +288,46 @@ class AudioRecognition {
           }
         }
       }
-      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();
-      await (0, import_utils.delay)(Math.max(extraSleep, 0), { signal: controller.signal });
+      let extraSleep = endpointingDelay;
+      if (lastSpeakingTime !== void 0) {
+        extraSleep += lastSpeakingTime - Date.now();
+      }
+      if (extraSleep > 0) {
+        await (0, import_utils.delay)(Math.max(extraSleep, 0), { signal: controller.signal });
+      }
       this.logger.debug({ transcript: this.audioTranscript }, "end of user turn");
+      const confidenceAvg = this.finalTranscriptConfidence.length > 0 ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) / this.finalTranscriptConfidence.length : 0;
+      let startedSpeakingAt;
+      let stoppedSpeakingAt;
+      let transcriptionDelay;
+      let endOfUtteranceDelay;
+      if (lastFinalTranscriptTime !== 0 && lastSpeakingTime !== void 0 && speechStartTime !== void 0) {
+        startedSpeakingAt = speechStartTime;
+        stoppedSpeakingAt = lastSpeakingTime;
+        transcriptionDelay = Math.max(lastFinalTranscriptTime - lastSpeakingTime, 0);
+        endOfUtteranceDelay = Date.now() - lastSpeakingTime;
+      }
       const committed = await this.hooks.onEndOfTurn({
         newTranscript: this.audioTranscript,
-        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),
-        endOfUtteranceDelay: Date.now() - lastSpeakingTime
+        transcriptConfidence: confidenceAvg,
+        transcriptionDelay: transcriptionDelay ?? 0,
+        endOfUtteranceDelay: endOfUtteranceDelay ?? 0,
+        startedSpeakingAt,
+        stoppedSpeakingAt
       });
       if (committed) {
         this.audioTranscript = "";
+        this.finalTranscriptConfidence = [];
+        this.lastSpeakingTime = void 0;
+        this.lastFinalTranscriptTime = 0;
+        this.speechStartTime = void 0;
       }
       this.userTurnCommitted = false;
     };
     (_a = this.bounceEOUTask) == null ? void 0 : _a.cancel();
-    this.bounceEOUTask = import_utils.Task.from(bounceEOUTask(this.lastSpeakingTime));
+    this.bounceEOUTask = import_utils.Task.from(
+      bounceEOUTask(this.lastSpeakingTime, this.lastFinalTranscriptTime, this.speechStartTime)
+    );
     this.bounceEOUTask.result.then(() => {
       this.logger.debug("EOU detection task completed");
     }).catch((err) => {
@@ -292,12 +407,17 @@ class AudioRecognition {
             break;
           case import_vad.VADEventType.INFERENCE_DONE:
             this.hooks.onVADInferenceDone(ev);
+            if (ev.rawAccumulatedSpeech > 0) {
+              this.lastSpeakingTime = Date.now();
+              if (this.speechStartTime === void 0) {
+                this.speechStartTime = Date.now();
+              }
+            }
             break;
           case import_vad.VADEventType.END_OF_SPEECH:
             this.logger.debug("VAD task: END_OF_SPEECH");
             this.hooks.onEndOfSpeech(ev);
             this.speaking = false;
-            this.lastSpeakingTime = Date.now() - ev.silenceDuration;
             if (this.vadBaseTurnDetection || this.turnDetectionMode === "stt" && this.userTurnCommitted) {
               const chatCtx = this.hooks.retrieveChatCtx();
               this.runEOUDetection(chatCtx);
@@ -321,6 +441,8 @@ class AudioRecognition {
     var _a;
     this.audioTranscript = "";
     this.audioInterimTranscript = "";
+    this.audioPreflightTranscript = "";
+    this.finalTranscriptConfidence = [];
     this.userTurnCommitted = false;
     (_a = this.sttTask) == null ? void 0 : _a.cancelAndWait().finally(() => {
       this.sttTask = import_utils.Task.from(({ signal }) => this.createSttTask(this.stt, signal));

package/dist/voice/audio_recognition.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/audio_recognition.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { WritableStreamDefaultWriter } from 'node:stream/web';\nimport { ReadableStream } from 'node:stream/web';\nimport { type ChatContext } from '../llm/chat_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream, isStreamReaderReleaseError } from '../stream/deferred_stream.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { mergeReadableStreams } from '../stream/merge_readable_streams.js';\nimport { type SpeechEvent, SpeechEventType } from '../stt/stt.js';\nimport { Task, delay } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { TurnDetectionMode } from './agent_session.js';\nimport type { STTNode } from './io.js';\n\nexport interface EndOfTurnInfo {\n  newTranscript: string;\n  transcriptionDelay: number;\n  endOfUtteranceDelay: number;\n}\n\nexport interface RecognitionHooks {\n  onStartOfSpeech: (ev: VADEvent) => void;\n  onVADInferenceDone: (ev: VADEvent) => void;\n  onEndOfSpeech: (ev: VADEvent) => void;\n  onInterimTranscript: (ev: SpeechEvent) => void;\n  onFinalTranscript: (ev: SpeechEvent) => void;\n  onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;\n\n  retrieveChatCtx: () => ChatContext;\n}\n\nexport interface _TurnDetector {\n  unlikelyThreshold: (language?: string) => Promise<number | undefined>;\n  supportsLanguage: (language?: string) => Promise<boolean>;\n  predictEndOfTurn(chatCtx: ChatContext): Promise<number>;\n}\n\nexport interface AudioRecognitionOptions {\n  recognitionHooks: RecognitionHooks;\n  stt?: STTNode;\n  vad?: VAD;\n  turnDetector?: _TurnDetector;\n  turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n}\n\nexport class AudioRecognition {\n  private hooks: RecognitionHooks;\n  private stt?: STTNode;\n  private vad?: VAD;\n  private turnDetector?: _TurnDetector;\n  private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  private minEndpointingDelay: number;\n  private maxEndpointingDelay: number;\n  private lastLanguage?: string;\n\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private lastFinalTranscriptTime = 0;\n  private audioTranscript = '';\n  private audioInterimTranscript = '';\n  private lastSpeakingTime = 0;\n  private userTurnCommitted = false;\n  private speaking = false;\n  private sampleRate?: number;\n\n  private vadInputStream: ReadableStream<AudioFrame>;\n  private sttInputStream: ReadableStream<AudioFrame>;\n  private silenceAudioTransform = new IdentityTransform<AudioFrame>();\n  private silenceAudioWriter: WritableStreamDefaultWriter<AudioFrame>;\n\n  // all cancellable tasks\n  private bounceEOUTask?: Task<void>;\n  private commitUserTurnTask?: Task<void>;\n  private vadTask?: Task<void>;\n  private sttTask?: Task<void>;\n\n  constructor(opts: AudioRecognitionOptions) {\n    this.hooks = opts.recognitionHooks;\n    this.stt = opts.stt;\n    this.vad = opts.vad;\n    this.turnDetector = opts.turnDetector;\n    this.turnDetectionMode = opts.turnDetectionMode;\n    this.minEndpointingDelay = opts.minEndpointingDelay;\n    this.maxEndpointingDelay = opts.maxEndpointingDelay;\n    this.lastLanguage = undefined;\n\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();\n    this.vadInputStream = vadInputStream;\n    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);\n    this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();\n  }\n\n  /**\n   * Current transcript of the user's speech, including interim transcript if available.\n   */\n  get currentTranscript(): string {\n    if (this.audioInterimTranscript) {\n      return `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n    }\n    return this.audioTranscript;\n  }\n\n  async start() {\n    this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));\n    this.vadTask.result.catch((err) => {\n      this.logger.error(`Error running VAD task: ${err}`);\n    });\n\n    this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n    this.sttTask.result.catch((err) => {\n      this.logger.error(`Error running STT task: ${err}`);\n    });\n  }\n\n  private async onSTTEvent(ev: SpeechEvent) {\n    if (\n      this.turnDetectionMode === 'manual' &&\n      this.userTurnCommitted &&\n      (this.bounceEOUTask === undefined ||\n        this.bounceEOUTask.done ||\n        ev.type == SpeechEventType.INTERIM_TRANSCRIPT)\n    ) {\n      // ignore stt event if user turn already committed and EOU task is done\n      // or it's an interim transcript\n      this.logger.debug(\n        {\n          userTurnCommitted: this.userTurnCommitted,\n          eouTaskDone: this.bounceEOUTask?.done,\n          evType: ev.type,\n          turnDetectionMode: this.turnDetectionMode,\n        },\n        'ignoring stt event',\n      );\n      return;\n    }\n\n    switch (ev.type) {\n      case SpeechEventType.FINAL_TRANSCRIPT:\n        this.hooks.onFinalTranscript(ev);\n        const transcript = ev.alternatives?.[0]?.text;\n        this.lastLanguage = ev.alternatives?.[0]?.language;\n\n        if (!transcript) {\n          // stt final transcript received but no transcript\n          return;\n        }\n\n        this.logger.debug(\n          {\n            user_transcript: transcript,\n            language: this.lastLanguage,\n          },\n          'received user transcript',\n        );\n\n        this.lastFinalTranscriptTime = Date.now();\n        this.audioTranscript += ` ${transcript}`;\n        this.audioTranscript = this.audioTranscript.trimStart();\n        this.audioInterimTranscript = '';\n\n        if (!this.speaking) {\n          if (!this.vad) {\n            // Copied from python agents:\n            // vad disabled, use stt timestamp\n            // TODO: this would screw up transcription latency metrics\n            // but we'll live with it for now.\n            // the correct way is to ensure STT fires SpeechEventType.END_OF_SPEECH\n            // and using that timestamp for _last_speaking_time\n            this.lastSpeakingTime = Date.now();\n          }\n\n          if (this.vadBaseTurnDetection || this.userTurnCommitted) {\n            const chatCtx = this.hooks.retrieveChatCtx();\n            this.logger.debug('running EOU detection on stt FINAL_TRANSCRIPT');\n            this.runEOUDetection(chatCtx);\n          }\n        }\n        break;\n      case SpeechEventType.INTERIM_TRANSCRIPT:\n        this.logger.debug({ transcript: ev.alternatives?.[0]?.text }, 'interim transcript');\n        this.hooks.onInterimTranscript(ev);\n        this.audioInterimTranscript = ev.alternatives?.[0]?.text ?? '';\n        break;\n      case SpeechEventType.END_OF_SPEECH:\n        if (this.turnDetectionMode !== 'stt') break;\n        this.userTurnCommitted = true;\n\n        if (!this.speaking) {\n          const chatCtx = this.hooks.retrieveChatCtx();\n          this.logger.debug('running EOU detection on stt END_OF_SPEECH');\n          this.runEOUDetection(chatCtx);\n        }\n    }\n  }\n\n  private runEOUDetection(chatCtx: ChatContext) {\n    this.logger.debug(\n      {\n        stt: this.stt,\n        audioTranscript: this.audioTranscript,\n        turnDetectionMode: this.turnDetectionMode,\n      },\n      'running EOU detection',\n    );\n\n    if (this.stt && !this.audioTranscript && this.turnDetectionMode !== 'manual') {\n      // stt enabled but no transcript yet\n      this.logger.debug('skipping EOU detection');\n      return;\n    }\n\n    chatCtx = chatCtx.copy();\n    chatCtx.addMessage({ role: 'user', content: this.audioTranscript });\n\n    const turnDetector =\n      // disable EOU model if manual turn detection enabled\n      this.audioTranscript && this.turnDetectionMode !== 'manual' ? this.turnDetector : undefined;\n\n    const bounceEOUTask = (lastSpeakingTime: number) => async (controller: AbortController) => {\n      let endpointingDelay = this.minEndpointingDelay;\n\n      // TODO(AJS-74): need to support actual turn detection model plugins for following code to run\n      if (turnDetector) {\n        this.logger.debug('Running turn detector model');\n        if (!turnDetector.supportsLanguage(this.lastLanguage)) {\n          this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);\n        } else {\n          const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);\n          this.logger.debug(\n            { endOfTurnProbability, language: this.lastLanguage },\n            'end of turn probability',\n          );\n\n          const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);\n          this.logger.debug(\n            {\n              unlikelyThreshold,\n              endOfTurnProbability,\n              language: this.lastLanguage,\n              transcript: this.audioTranscript,\n            },\n            'EOU Detection',\n          );\n\n          if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {\n            endpointingDelay = this.maxEndpointingDelay;\n          }\n        }\n      }\n\n      const extraSleep = lastSpeakingTime + endpointingDelay - Date.now();\n      // add delay to see if there's a potential upcoming EOU task that cancels this one\n      await delay(Math.max(extraSleep, 0), { signal: controller.signal });\n\n      this.logger.debug({ transcript: this.audioTranscript }, 'end of user turn');\n\n      const committed = await this.hooks.onEndOfTurn({\n        newTranscript: this.audioTranscript,\n        transcriptionDelay: Math.max(this.lastFinalTranscriptTime - lastSpeakingTime, 0),\n        endOfUtteranceDelay: Date.now() - lastSpeakingTime,\n      });\n\n      if (committed) {\n        // clear the transcript if the user turn was committed\n        this.audioTranscript = '';\n      }\n\n      this.userTurnCommitted = false;\n    };\n\n    // cancel any existing EOU task\n    this.bounceEOUTask?.cancel();\n    this.bounceEOUTask = Task.from(bounceEOUTask(this.lastSpeakingTime));\n\n    this.bounceEOUTask.result\n      .then(() => {\n        this.logger.debug('EOU detection task completed');\n      })\n      .catch((err: unknown) => {\n        if (err instanceof Error && err.message.includes('This operation was aborted')) {\n          // ignore aborted errors\n          return;\n        }\n        this.logger.error(err, 'Error in EOU detection task:');\n      });\n  }\n\n  private async createSttTask(stt: STTNode | undefined, signal: AbortSignal) {\n    if (!stt) return;\n\n    this.logger.debug('createSttTask: create stt stream from stt node');\n\n    const sttStream = await stt(this.sttInputStream, {});\n\n    if (signal.aborted || sttStream === null) return;\n\n    if (sttStream instanceof ReadableStream) {\n      const reader = sttStream.getReader();\n\n      signal.addEventListener('abort', async () => {\n        try {\n          reader.releaseLock();\n          await sttStream?.cancel();\n        } catch (e) {\n          this.logger.debug('createSttTask: error during abort handler:', e);\n        }\n      });\n\n      try {\n        while (true) {\n          if (signal.aborted) break;\n\n          const { done, value: ev } = await reader.read();\n          if (done) break;\n\n          if (typeof ev === 'string') {\n            throw new Error('STT node must yield SpeechEvent');\n          } else {\n            await this.onSTTEvent(ev);\n          }\n        }\n      } catch (e) {\n        if (isStreamReaderReleaseError(e)) {\n          return;\n        }\n        this.logger.error({ error: e }, 'createSttTask: error reading sttStream');\n      } finally {\n        reader.releaseLock();\n        try {\n          await sttStream.cancel();\n        } catch (e) {\n          this.logger.debug(\n            'createSttTask: error cancelling sttStream (may already be cancelled):',\n            e,\n          );\n        }\n      }\n    }\n  }\n\n  private async createVadTask(vad: VAD | undefined, signal: AbortSignal) {\n    if (!vad) return;\n\n    const vadStream = vad.stream();\n    vadStream.updateInputStream(this.vadInputStream);\n\n    const abortHandler = () => {\n      vadStream.detachInputStream();\n      vadStream.close();\n      signal.removeEventListener('abort', abortHandler);\n    };\n    signal.addEventListener('abort', abortHandler);\n\n    try {\n      for await (const ev of vadStream) {\n        if (signal.aborted) break;\n\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.logger.debug('VAD task: START_OF_SPEECH');\n            this.hooks.onStartOfSpeech(ev);\n            this.speaking = true;\n\n            // Capture sample rate from the first VAD event if not already set\n            if (ev.frames.length > 0 && ev.frames[0]) {\n              this.sampleRate = ev.frames[0].sampleRate;\n            }\n\n            this.bounceEOUTask?.cancel();\n            break;\n          case VADEventType.INFERENCE_DONE:\n            this.hooks.onVADInferenceDone(ev);\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.logger.debug('VAD task: END_OF_SPEECH');\n            this.hooks.onEndOfSpeech(ev);\n            this.speaking = false;\n            // when VAD fires END_OF_SPEECH, it already waited for the silence_duration\n            this.lastSpeakingTime = Date.now() - ev.silenceDuration;\n\n            if (\n              this.vadBaseTurnDetection ||\n              (this.turnDetectionMode === 'stt' && this.userTurnCommitted)\n            ) {\n              const chatCtx = this.hooks.retrieveChatCtx();\n              this.runEOUDetection(chatCtx);\n            }\n            break;\n        }\n      }\n    } catch (e) {\n      this.logger.error(e, 'Error in VAD task');\n    } finally {\n      this.logger.debug('VAD task closed');\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputAudioStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  clearUserTurn() {\n    this.audioTranscript = '';\n    this.audioInterimTranscript = '';\n    this.userTurnCommitted = false;\n\n    this.sttTask?.cancelAndWait().finally(() => {\n      this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n      this.sttTask.result.catch((err) => {\n        this.logger.error(`Error running STT task: ${err}`);\n      });\n    });\n  }\n\n  commitUserTurn(audioDetached: boolean) {\n    const commitUserTurnTask =\n      (delayDuration: number = 500) =>\n      async (controller: AbortController) => {\n        if (Date.now() - this.lastFinalTranscriptTime > delayDuration) {\n          // flush the stt by pushing silence\n          if (audioDetached && this.sampleRate !== undefined) {\n            const numSamples = Math.floor(this.sampleRate * 0.5);\n            const silence = new Int16Array(numSamples * 2);\n            const silenceFrame = new AudioFrame(silence, this.sampleRate, 1, numSamples);\n            this.silenceAudioWriter.write(silenceFrame);\n          }\n\n          // wait for the final transcript to be available\n          await delay(delayDuration, { signal: controller.signal });\n        }\n\n        if (this.audioInterimTranscript) {\n          // append interim transcript in case the final transcript is not ready\n          this.audioTranscript = `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n        }\n        this.audioInterimTranscript = '';\n\n        const chatCtx = this.hooks.retrieveChatCtx();\n        this.logger.debug('running EOU detection on commitUserTurn');\n        this.runEOUDetection(chatCtx);\n        this.userTurnCommitted = true;\n      };\n\n    // cancel any existing commit user turn task\n    this.commitUserTurnTask?.cancel();\n    this.commitUserTurnTask = Task.from(commitUserTurnTask());\n\n    this.commitUserTurnTask.result\n      .then(() => {\n        this.logger.debug('User turn committed');\n      })\n      .catch((err: unknown) => {\n        this.logger.error(err, 'Error in user turn commit task:');\n      });\n  }\n\n  async close() {\n    this.detachInputAudioStream();\n    await this.commitUserTurnTask?.cancelAndWait();\n    await this.sttTask?.cancelAndWait();\n    await this.vadTask?.cancelAndWait();\n    await this.bounceEOUTask?.cancelAndWait();\n  }\n\n  private get vadBaseTurnDetection() {\n    return ['vad', undefined].includes(this.turnDetectionMode);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA2B;AAE3B,iBAA+B;AAC/B,0BAAiC;AACjC,iBAAoB;AACpB,6BAAmE;AACnE,gCAAkC;AAClC,oCAAqC;AACrC,iBAAkD;AAClD,mBAA4B;AAC5B,iBAAsD;AAqC/C,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA,aAAS,gBAAI;AAAA,EACb,0BAA0B;AAAA,EAC1B,kBAAkB;AAAA,EAClB,yBAAyB;AAAA,EACzB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,WAAW;AAAA,EACX;AAAA,EAEA;AAAA,EACA;AAAA,EACA,wBAAwB,IAAI,4CAA8B;AAAA,EAC1D;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,MAA+B;AACzC,SAAK,QAAQ,KAAK;AAClB,SAAK,MAAM,KAAK;AAChB,SAAK,MAAM,KAAK;AAChB,SAAK,eAAe,KAAK;AACzB,SAAK,oBAAoB,KAAK;AAC9B,SAAK,sBAAsB,KAAK;AAChC,SAAK,sBAAsB,KAAK;AAChC,SAAK,eAAe;AAEpB,SAAK,sBAAsB,IAAI,8CAAmC;AAClE,UAAM,CAAC,gBAAgB,cAAc,IAAI,KAAK,oBAAoB,OAAO,IAAI;AAC7E,SAAK,iBAAiB;AACtB,SAAK,qBAAiB,oDAAqB,gBAAgB,KAAK,sBAAsB,QAAQ;AAC9F,SAAK,qBAAqB,KAAK,sBAAsB,SAAS,UAAU;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,oBAA4B;AAC9B,QAAI,KAAK,wBAAwB;AAC/B,aAAO,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,IACvE;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAED,SAAK,UAAU,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,WAAW,IAAiB;AAxH5C;AAyHI,QACE,KAAK,sBAAsB,YAC3B,KAAK,sBACJ,KAAK,kBAAkB,UACtB,KAAK,cAAc,QACnB,GAAG,QAAQ,2BAAgB,qBAC7B;AAGA,WAAK,OAAO;AAAA,QACV;AAAA,UACE,mBAAmB,KAAK;AAAA,UACxB,cAAa,UAAK,kBAAL,mBAAoB;AAAA,UACjC,QAAQ,GAAG;AAAA,UACX,mBAAmB,KAAK;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AACA;AAAA,IACF;AAEA,YAAQ,GAAG,MAAM;AAAA,MACf,KAAK,2BAAgB;AACnB,aAAK,MAAM,kBAAkB,EAAE;AAC/B,cAAM,cAAa,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AACzC,aAAK,gBAAe,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AAE1C,YAAI,CAAC,YAAY;AAEf;AAAA,QACF;AAEA,aAAK,OAAO;AAAA,UACV;AAAA,YACE,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,UACjB;AAAA,UACA;AAAA,QACF;AAEA,aAAK,0BAA0B,KAAK,IAAI;AACxC,aAAK,mBAAmB,IAAI,UAAU;AACtC,aAAK,kBAAkB,KAAK,gBAAgB,UAAU;AACtD,aAAK,yBAAyB;AAE9B,YAAI,CAAC,KAAK,UAAU;AAClB,cAAI,CAAC,KAAK,KAAK;AAOb,iBAAK,mBAAmB,KAAK,IAAI;AAAA,UACnC;AAEA,cAAI,KAAK,wBAAwB,KAAK,mBAAmB;AACvD,kBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,iBAAK,OAAO,MAAM,+CAA+C;AACjE,iBAAK,gBAAgB,OAAO;AAAA,UAC9B;AAAA,QACF;AACA;AAAA,MACF,KAAK,2BAAgB;AACnB,aAAK,OAAO,MAAM,EAAE,aAAY,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,KAAK,GAAG,oBAAoB;AAClF,aAAK,MAAM,oBAAoB,EAAE;AACjC,aAAK,2BAAyB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,SAAQ;AAC5D;AAAA,MACF,KAAK,2BAAgB;AACnB,YAAI,KAAK,sBAAsB,MAAO;AACtC,aAAK,oBAAoB;AAEzB,YAAI,CAAC,KAAK,UAAU;AAClB,gBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,eAAK,OAAO,MAAM,4CAA4C;AAC9D,eAAK,gBAAgB,OAAO;AAAA,QAC9B;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,gBAAgB,SAAsB;AAzMhD;AA0MI,SAAK,OAAO;AAAA,MACV;AAAA,QACE,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,mBAAmB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,CAAC,KAAK,mBAAmB,KAAK,sBAAsB,UAAU;AAE5E,WAAK,OAAO,MAAM,wBAAwB;AAC1C;AAAA,IACF;AAEA,cAAU,QAAQ,KAAK;AACvB,YAAQ,WAAW,EAAE,MAAM,QAAQ,SAAS,KAAK,gBAAgB,CAAC;AAElE,UAAM;AAAA;AAAA,MAEJ,KAAK,mBAAmB,KAAK,sBAAsB,WAAW,KAAK,eAAe;AAAA;AAEpF,UAAM,gBAAgB,CAAC,qBAA6B,OAAO,eAAgC;AACzF,UAAI,mBAAmB,KAAK;AAG5B,UAAI,cAAc;AAChB,aAAK,OAAO,MAAM,6BAA6B;AAC/C,YAAI,CAAC,aAAa,iBAAiB,KAAK,YAAY,GAAG;AACrD,eAAK,OAAO,MAAM,2CAA2C,KAAK,YAAY,EAAE;AAAA,QAClF,OAAO;AACL,gBAAM,uBAAuB,MAAM,aAAa,iBAAiB,OAAO;AACxE,eAAK,OAAO;AAAA,YACV,EAAE,sBAAsB,UAAU,KAAK,aAAa;AAAA,YACpD;AAAA,UACF;AAEA,gBAAM,oBAAoB,MAAM,aAAa,kBAAkB,KAAK,YAAY;AAChF,eAAK,OAAO;AAAA,YACV;AAAA,cACE;AAAA,cACA;AAAA,cACA,UAAU,KAAK;AAAA,cACf,YAAY,KAAK;AAAA,YACnB;AAAA,YACA;AAAA,UACF;AAEA,cAAI,qBAAqB,uBAAuB,mBAAmB;AACjE,+BAAmB,KAAK;AAAA,UAC1B;AAAA,QACF;AAAA,MACF;AAEA,YAAM,aAAa,mBAAmB,mBAAmB,KAAK,IAAI;AAElE,gBAAM,oBAAM,KAAK,IAAI,YAAY,CAAC,GAAG,EAAE,QAAQ,WAAW,OAAO,CAAC;AAElE,WAAK,OAAO,MAAM,EAAE,YAAY,KAAK,gBAAgB,GAAG,kBAAkB;AAE1E,YAAM,YAAY,MAAM,KAAK,MAAM,YAAY;AAAA,QAC7C,eAAe,KAAK;AAAA,QACpB,oBAAoB,KAAK,IAAI,KAAK,0BAA0B,kBAAkB,CAAC;AAAA,QAC/E,qBAAqB,KAAK,IAAI,IAAI;AAAA,MACpC,CAAC;AAED,UAAI,WAAW;AAEb,aAAK,kBAAkB;AAAA,MACzB;AAEA,WAAK,oBAAoB;AAAA,IAC3B;AAGA,eAAK,kBAAL,mBAAoB;AACpB,SAAK,gBAAgB,kBAAK,KAAK,cAAc,KAAK,gBAAgB,CAAC;AAEnE,SAAK,cAAc,OAChB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,8BAA8B;AAAA,IAClD,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,UAAI,eAAe,SAAS,IAAI,QAAQ,SAAS,4BAA4B,GAAG;AAE9E;AAAA,MACF;AACA,WAAK,OAAO,MAAM,KAAK,8BAA8B;AAAA,IACvD,CAAC;AAAA,EACL;AAAA,EAEA,MAAc,cAAc,KAA0B,QAAqB;AACzE,QAAI,CAAC,IAAK;AAEV,SAAK,OAAO,MAAM,gDAAgD;AAElE,UAAM,YAAY,MAAM,IAAI,KAAK,gBAAgB,CAAC,CAAC;AAEnD,QAAI,OAAO,WAAW,cAAc,KAAM;AAE1C,QAAI,qBAAqB,2BAAgB;AACvC,YAAM,SAAS,UAAU,UAAU;AAEnC,aAAO,iBAAiB,SAAS,YAAY;AAC3C,YAAI;AACF,iBAAO,YAAY;AACnB,iBAAM,uCAAW;AAAA,QACnB,SAAS,GAAG;AACV,eAAK,OAAO,MAAM,8CAA8C,CAAC;AAAA,QACnE;AAAA,MACF,CAAC;AAED,UAAI;AACF,eAAO,MAAM;AACX,cAAI,OAAO,QAAS;AAEpB,gBAAM,EAAE,MAAM,OAAO,GAAG,IAAI,MAAM,OAAO,KAAK;AAC9C,cAAI,KAAM;AAEV,cAAI,OAAO,OAAO,UAAU;AAC1B,kBAAM,IAAI,MAAM,iCAAiC;AAAA,UACnD,OAAO;AACL,kBAAM,KAAK,WAAW,EAAE;AAAA,UAC1B;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,gBAAI,mDAA2B,CAAC,GAAG;AACjC;AAAA,QACF;AACA,aAAK,OAAO,MAAM,EAAE,OAAO,EAAE,GAAG,wCAAwC;AAAA,MAC1E,UAAE;AACA,eAAO,YAAY;AACnB,YAAI;AACF,gBAAM,UAAU,OAAO;AAAA,QACzB,SAAS,GAAG;AACV,eAAK,OAAO;AAAA,YACV;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,KAAsB,QAAqB;AA1VzE;AA2VI,QAAI,CAAC,IAAK;AAEV,UAAM,YAAY,IAAI,OAAO;AAC7B,cAAU,kBAAkB,KAAK,cAAc;AAE/C,UAAM,eAAe,MAAM;AACzB,gBAAU,kBAAkB;AAC5B,gBAAU,MAAM;AAChB,aAAO,oBAAoB,SAAS,YAAY;AAAA,IAClD;AACA,WAAO,iBAAiB,SAAS,YAAY;AAE7C,QAAI;AACF,uBAAiB,MAAM,WAAW;AAChC,YAAI,OAAO,QAAS;AAEpB,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,wBAAa;AAChB,iBAAK,OAAO,MAAM,2BAA2B;AAC7C,iBAAK,MAAM,gBAAgB,EAAE;AAC7B,iBAAK,WAAW;AAGhB,gBAAI,GAAG,OAAO,SAAS,KAAK,GAAG,OAAO,CAAC,GAAG;AACxC,mBAAK,aAAa,GAAG,OAAO,CAAC,EAAE;AAAA,YACjC;AAEA,uBAAK,kBAAL,mBAAoB;AACpB;AAAA,UACF,KAAK,wBAAa;AAChB,iBAAK,MAAM,mBAAmB,EAAE;AAChC;AAAA,UACF,KAAK,wBAAa;AAChB,iBAAK,OAAO,MAAM,yBAAyB;AAC3C,iBAAK,MAAM,cAAc,EAAE;AAC3B,iBAAK,WAAW;AAEhB,iBAAK,mBAAmB,KAAK,IAAI,IAAI,GAAG;AAExC,gBACE,KAAK,wBACJ,KAAK,sBAAsB,SAAS,KAAK,mBAC1C;AACA,oBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,mBAAK,gBAAgB,OAAO;AAAA,YAC9B;AACA;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,GAAG,mBAAmB;AAAA,IAC1C,UAAE;AACA,WAAK,OAAO,MAAM,iBAAiB;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,oBAAoB,aAAyC;AAC3D,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,yBAAyB;AACvB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA,EAEA,gBAAgB;AA3ZlB;AA4ZI,SAAK,kBAAkB;AACvB,SAAK,yBAAyB;AAC9B,SAAK,oBAAoB;AAEzB,eAAK,YAAL,mBAAc,gBAAgB,QAAQ,MAAM;AAC1C,WAAK,UAAU,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,WAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,aAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,MACpD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,eAAe,eAAwB;AAxazC;AAyaI,UAAM,qBACJ,CAAC,gBAAwB,QACzB,OAAO,eAAgC;AACrC,UAAI,KAAK,IAAI,IAAI,KAAK,0BAA0B,eAAe;AAE7D,YAAI,iBAAiB,KAAK,eAAe,QAAW;AAClD,gBAAM,aAAa,KAAK,MAAM,KAAK,aAAa,GAAG;AACnD,gBAAM,UAAU,IAAI,WAAW,aAAa,CAAC;AAC7C,gBAAM,eAAe,IAAI,2BAAW,SAAS,KAAK,YAAY,GAAG,UAAU;AAC3E,eAAK,mBAAmB,MAAM,YAAY;AAAA,QAC5C;AAGA,kBAAM,oBAAM,eAAe,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,MAC1D;AAEA,UAAI,KAAK,wBAAwB;AAE/B,aAAK,kBAAkB,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,MACvF;AACA,WAAK,yBAAyB;AAE9B,YAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,WAAK,OAAO,MAAM,yCAAyC;AAC3D,WAAK,gBAAgB,OAAO;AAC5B,WAAK,oBAAoB;AAAA,IAC3B;AAGF,eAAK,uBAAL,mBAAyB;AACzB,SAAK,qBAAqB,kBAAK,KAAK,mBAAmB,CAAC;AAExD,SAAK,mBAAmB,OACrB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,qBAAqB;AAAA,IACzC,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,WAAK,OAAO,MAAM,KAAK,iCAAiC;AAAA,IAC1D,CAAC;AAAA,EACL;AAAA,EAEA,MAAM,QAAQ;AAldhB;AAmdI,SAAK,uBAAuB;AAC5B,YAAM,UAAK,uBAAL,mBAAyB;AAC/B,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,kBAAL,mBAAoB;AAAA,EAC5B;AAAA,EAEA,IAAY,uBAAuB;AACjC,WAAO,CAAC,OAAO,MAAS,EAAE,SAAS,KAAK,iBAAiB;AAAA,EAC3D;AACF;","names":[]}
+{"version":3,"sources":["../../src/voice/audio_recognition.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { WritableStreamDefaultWriter } from 'node:stream/web';\nimport { ReadableStream } from 'node:stream/web';\nimport { type ChatContext } from '../llm/chat_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream, isStreamReaderReleaseError } from '../stream/deferred_stream.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { mergeReadableStreams } from '../stream/merge_readable_streams.js';\nimport { type SpeechEvent, SpeechEventType } from '../stt/stt.js';\nimport { Task, delay } from '../utils.js';\nimport { type VAD, type VADEvent, VADEventType } from '../vad.js';\nimport type { TurnDetectionMode } from './agent_session.js';\nimport type { STTNode } from './io.js';\n\nexport interface EndOfTurnInfo {\n  newTranscript: string;\n  transcriptConfidence: number;\n  transcriptionDelay: number;\n  endOfUtteranceDelay: number;\n  startedSpeakingAt: number | undefined;\n  stoppedSpeakingAt: number | undefined;\n}\n\nexport interface PreemptiveGenerationInfo {\n  newTranscript: string;\n  transcriptConfidence: number;\n}\n\nexport interface RecognitionHooks {\n  onStartOfSpeech: (ev: VADEvent) => void;\n  onVADInferenceDone: (ev: VADEvent) => void;\n  onEndOfSpeech: (ev: VADEvent) => void;\n  onInterimTranscript: (ev: SpeechEvent) => void;\n  onFinalTranscript: (ev: SpeechEvent) => void;\n  onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;\n  onPreemptiveGeneration: (info: PreemptiveGenerationInfo) => void;\n\n  retrieveChatCtx: () => ChatContext;\n}\n\nexport interface _TurnDetector {\n  unlikelyThreshold: (language?: string) => Promise<number | undefined>;\n  supportsLanguage: (language?: string) => Promise<boolean>;\n  predictEndOfTurn(chatCtx: ChatContext): Promise<number>;\n}\n\nexport interface AudioRecognitionOptions {\n  recognitionHooks: RecognitionHooks;\n  stt?: STTNode;\n  vad?: VAD;\n  turnDetector?: _TurnDetector;\n  turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n}\n\nexport class AudioRecognition {\n  private hooks: RecognitionHooks;\n  private stt?: STTNode;\n  private vad?: VAD;\n  private turnDetector?: _TurnDetector;\n  private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;\n  private minEndpointingDelay: number;\n  private maxEndpointingDelay: number;\n  private lastLanguage?: string;\n\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private lastFinalTranscriptTime = 0;\n  private audioTranscript = '';\n  private audioInterimTranscript = '';\n  private audioPreflightTranscript = '';\n  private finalTranscriptConfidence: number[] = [];\n  private lastSpeakingTime: number | undefined;\n  private speechStartTime: number | undefined;\n  private userTurnCommitted = false;\n  private speaking = false;\n  private sampleRate?: number;\n\n  private vadInputStream: ReadableStream<AudioFrame>;\n  private sttInputStream: ReadableStream<AudioFrame>;\n  private silenceAudioTransform = new IdentityTransform<AudioFrame>();\n  private silenceAudioWriter: WritableStreamDefaultWriter<AudioFrame>;\n\n  // all cancellable tasks\n  private bounceEOUTask?: Task<void>;\n  private commitUserTurnTask?: Task<void>;\n  private vadTask?: Task<void>;\n  private sttTask?: Task<void>;\n\n  constructor(opts: AudioRecognitionOptions) {\n    this.hooks = opts.recognitionHooks;\n    this.stt = opts.stt;\n    this.vad = opts.vad;\n    this.turnDetector = opts.turnDetector;\n    this.turnDetectionMode = opts.turnDetectionMode;\n    this.minEndpointingDelay = opts.minEndpointingDelay;\n    this.maxEndpointingDelay = opts.maxEndpointingDelay;\n    this.lastLanguage = undefined;\n\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();\n    this.vadInputStream = vadInputStream;\n    this.sttInputStream = mergeReadableStreams(sttInputStream, this.silenceAudioTransform.readable);\n    this.silenceAudioWriter = this.silenceAudioTransform.writable.getWriter();\n  }\n\n  /**\n   * Current transcript of the user's speech, including interim transcript if available.\n   */\n  get currentTranscript(): string {\n    if (this.audioInterimTranscript) {\n      return `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n    }\n    return this.audioTranscript;\n  }\n\n  async start() {\n    this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));\n    this.vadTask.result.catch((err) => {\n      this.logger.error(`Error running VAD task: ${err}`);\n    });\n\n    this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n    this.sttTask.result.catch((err) => {\n      this.logger.error(`Error running STT task: ${err}`);\n    });\n  }\n\n  private async onSTTEvent(ev: SpeechEvent) {\n    if (\n      this.turnDetectionMode === 'manual' &&\n      this.userTurnCommitted &&\n      (this.bounceEOUTask === undefined ||\n        this.bounceEOUTask.done ||\n        ev.type == SpeechEventType.INTERIM_TRANSCRIPT)\n    ) {\n      // ignore stt event if user turn already committed and EOU task is done\n      // or it's an interim transcript\n      this.logger.debug(\n        {\n          userTurnCommitted: this.userTurnCommitted,\n          eouTaskDone: this.bounceEOUTask?.done,\n          evType: ev.type,\n          turnDetectionMode: this.turnDetectionMode,\n        },\n        'ignoring stt event',\n      );\n      return;\n    }\n\n    switch (ev.type) {\n      case SpeechEventType.FINAL_TRANSCRIPT:\n        this.hooks.onFinalTranscript(ev);\n        const transcript = ev.alternatives?.[0]?.text;\n        const confidence = ev.alternatives?.[0]?.confidence ?? 0;\n        this.lastLanguage = ev.alternatives?.[0]?.language;\n\n        if (!transcript) {\n          // stt final transcript received but no transcript\n          return;\n        }\n\n        this.logger.debug(\n          {\n            user_transcript: transcript,\n            language: this.lastLanguage,\n          },\n          'received user transcript',\n        );\n\n        this.lastFinalTranscriptTime = Date.now();\n        this.audioTranscript += ` ${transcript}`;\n        this.audioTranscript = this.audioTranscript.trimStart();\n        this.finalTranscriptConfidence.push(confidence);\n        const transcriptChanged = this.audioTranscript !== this.audioPreflightTranscript;\n        this.audioInterimTranscript = '';\n        this.audioPreflightTranscript = '';\n\n        if (!this.vad || this.lastSpeakingTime === undefined) {\n          // vad disabled, use stt timestamp\n          // TODO: this would screw up transcription latency metrics\n          // but we'll live with it for now.\n          // the correct way is to ensure STT fires SpeechEventType.END_OF_SPEECH\n          // and using that timestamp for lastSpeakingTime\n          this.lastSpeakingTime = Date.now();\n        }\n\n        if (this.vadBaseTurnDetection || this.userTurnCommitted) {\n          if (transcriptChanged) {\n            this.logger.debug(\n              { transcript: this.audioTranscript },\n              'triggering preemptive generation (FINAL_TRANSCRIPT)',\n            );\n            this.hooks.onPreemptiveGeneration({\n              newTranscript: this.audioTranscript,\n              transcriptConfidence:\n                this.finalTranscriptConfidence.length > 0\n                  ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) /\n                    this.finalTranscriptConfidence.length\n                  : 0,\n            });\n          }\n\n          if (!this.speaking) {\n            const chatCtx = this.hooks.retrieveChatCtx();\n            this.logger.debug('running EOU detection on stt FINAL_TRANSCRIPT');\n            this.runEOUDetection(chatCtx);\n          }\n        }\n        break;\n      case SpeechEventType.PREFLIGHT_TRANSCRIPT:\n        this.hooks.onInterimTranscript(ev);\n        const preflightTranscript = ev.alternatives?.[0]?.text ?? '';\n        const preflightConfidence = ev.alternatives?.[0]?.confidence ?? 0;\n        const preflightLanguage = ev.alternatives?.[0]?.language;\n\n        const MIN_LANGUAGE_DETECTION_LENGTH = 5;\n        if (\n          !this.lastLanguage ||\n          (preflightLanguage && preflightTranscript.length > MIN_LANGUAGE_DETECTION_LENGTH)\n        ) {\n          this.lastLanguage = preflightLanguage;\n        }\n\n        if (!preflightTranscript) {\n          return;\n        }\n\n        this.logger.debug(\n          {\n            user_transcript: preflightTranscript,\n            language: this.lastLanguage,\n          },\n          'received user preflight transcript',\n        );\n\n        // still need to increment it as it's used for turn detection,\n        this.lastFinalTranscriptTime = Date.now();\n        // preflight transcript includes all pre-committed transcripts (including final transcript from the previous STT run)\n        this.audioPreflightTranscript =\n          `${this.audioTranscript} ${preflightTranscript}`.trimStart();\n        this.audioInterimTranscript = preflightTranscript;\n\n        if (!this.vad || this.lastSpeakingTime === undefined) {\n          // vad disabled, use stt timestamp\n          this.lastSpeakingTime = Date.now();\n        }\n\n        if (this.turnDetectionMode !== 'manual' || this.userTurnCommitted) {\n          const confidenceVals = [...this.finalTranscriptConfidence, preflightConfidence];\n          this.logger.debug(\n            {\n              transcript:\n                this.audioPreflightTranscript.length > 100\n                  ? this.audioPreflightTranscript.slice(0, 100) + '...'\n                  : this.audioPreflightTranscript,\n            },\n            'triggering preemptive generation (PREFLIGHT_TRANSCRIPT)',\n          );\n          this.hooks.onPreemptiveGeneration({\n            newTranscript: this.audioPreflightTranscript,\n            transcriptConfidence:\n              confidenceVals.length > 0\n                ? confidenceVals.reduce((a, b) => a + b, 0) / confidenceVals.length\n                : 0,\n          });\n        }\n        break;\n      case SpeechEventType.INTERIM_TRANSCRIPT:\n        this.logger.debug({ transcript: ev.alternatives?.[0]?.text }, 'interim transcript');\n        this.hooks.onInterimTranscript(ev);\n        this.audioInterimTranscript = ev.alternatives?.[0]?.text ?? '';\n        break;\n      case SpeechEventType.START_OF_SPEECH:\n        if (this.turnDetectionMode !== 'stt') break;\n        this.hooks.onStartOfSpeech({\n          type: VADEventType.START_OF_SPEECH,\n          samplesIndex: 0,\n          timestamp: Date.now(),\n          speechDuration: 0,\n          silenceDuration: 0,\n          frames: [],\n          probability: 0,\n          inferenceDuration: 0,\n          speaking: true,\n          rawAccumulatedSilence: 0,\n          rawAccumulatedSpeech: 0,\n        });\n        this.speaking = true;\n        this.lastSpeakingTime = Date.now();\n\n        this.bounceEOUTask?.cancel();\n        break;\n      case SpeechEventType.END_OF_SPEECH:\n        if (this.turnDetectionMode !== 'stt') break;\n        this.hooks.onEndOfSpeech({\n          type: VADEventType.END_OF_SPEECH,\n          samplesIndex: 0,\n          timestamp: Date.now(),\n          speechDuration: 0,\n          silenceDuration: 0,\n          frames: [],\n          probability: 0,\n          inferenceDuration: 0,\n          speaking: false,\n          rawAccumulatedSilence: 0,\n          rawAccumulatedSpeech: 0,\n        });\n        this.speaking = false;\n        this.userTurnCommitted = true;\n        this.lastSpeakingTime = Date.now();\n\n        if (!this.speaking) {\n          const chatCtx = this.hooks.retrieveChatCtx();\n          this.logger.debug('running EOU detection on stt END_OF_SPEECH');\n          this.runEOUDetection(chatCtx);\n        }\n    }\n  }\n\n  private runEOUDetection(chatCtx: ChatContext) {\n    this.logger.debug(\n      {\n        stt: this.stt,\n        audioTranscript: this.audioTranscript,\n        turnDetectionMode: this.turnDetectionMode,\n      },\n      'running EOU detection',\n    );\n\n    if (this.stt && !this.audioTranscript && this.turnDetectionMode !== 'manual') {\n      // stt enabled but no transcript yet\n      this.logger.debug('skipping EOU detection');\n      return;\n    }\n\n    chatCtx = chatCtx.copy();\n    chatCtx.addMessage({ role: 'user', content: this.audioTranscript });\n\n    const turnDetector =\n      // disable EOU model if manual turn detection enabled\n      this.audioTranscript && this.turnDetectionMode !== 'manual' ? this.turnDetector : undefined;\n\n    const bounceEOUTask =\n      (\n        lastSpeakingTime: number | undefined,\n        lastFinalTranscriptTime: number,\n        speechStartTime: number | undefined,\n      ) =>\n      async (controller: AbortController) => {\n        let endpointingDelay = this.minEndpointingDelay;\n\n        if (turnDetector) {\n          this.logger.debug('Running turn detector model');\n          if (!turnDetector.supportsLanguage(this.lastLanguage)) {\n            this.logger.debug(`Turn detector does not support language ${this.lastLanguage}`);\n          } else {\n            const endOfTurnProbability = await turnDetector.predictEndOfTurn(chatCtx);\n            this.logger.debug(\n              { endOfTurnProbability, language: this.lastLanguage },\n              'end of turn probability',\n            );\n\n            const unlikelyThreshold = await turnDetector.unlikelyThreshold(this.lastLanguage);\n            this.logger.debug(\n              {\n                unlikelyThreshold,\n                endOfTurnProbability,\n                language: this.lastLanguage,\n                transcript: this.audioTranscript,\n              },\n              'EOU Detection',\n            );\n\n            if (unlikelyThreshold && endOfTurnProbability < unlikelyThreshold) {\n              endpointingDelay = this.maxEndpointingDelay;\n            }\n          }\n        }\n\n        let extraSleep = endpointingDelay;\n        if (lastSpeakingTime !== undefined) {\n          extraSleep += lastSpeakingTime - Date.now();\n        }\n\n        if (extraSleep > 0) {\n          // add delay to see if there's a potential upcoming EOU task that cancels this one\n          await delay(Math.max(extraSleep, 0), { signal: controller.signal });\n        }\n\n        this.logger.debug({ transcript: this.audioTranscript }, 'end of user turn');\n\n        const confidenceAvg =\n          this.finalTranscriptConfidence.length > 0\n            ? this.finalTranscriptConfidence.reduce((a, b) => a + b, 0) /\n              this.finalTranscriptConfidence.length\n            : 0;\n\n        let startedSpeakingAt: number | undefined;\n        let stoppedSpeakingAt: number | undefined;\n        let transcriptionDelay: number | undefined;\n        let endOfUtteranceDelay: number | undefined;\n\n        // sometimes, we can't calculate the metrics because VAD was unreliable.\n        // in this case, we just ignore the calculation, it's better than providing likely wrong values\n        if (\n          lastFinalTranscriptTime !== 0 &&\n          lastSpeakingTime !== undefined &&\n          speechStartTime !== undefined\n        ) {\n          startedSpeakingAt = speechStartTime;\n          stoppedSpeakingAt = lastSpeakingTime;\n          transcriptionDelay = Math.max(lastFinalTranscriptTime - lastSpeakingTime, 0);\n          endOfUtteranceDelay = Date.now() - lastSpeakingTime;\n        }\n\n        const committed = await this.hooks.onEndOfTurn({\n          newTranscript: this.audioTranscript,\n          transcriptConfidence: confidenceAvg,\n          transcriptionDelay: transcriptionDelay ?? 0,\n          endOfUtteranceDelay: endOfUtteranceDelay ?? 0,\n          startedSpeakingAt,\n          stoppedSpeakingAt,\n        });\n\n        if (committed) {\n          // clear the transcript if the user turn was committed\n          this.audioTranscript = '';\n          this.finalTranscriptConfidence = [];\n          this.lastSpeakingTime = undefined;\n          this.lastFinalTranscriptTime = 0;\n          this.speechStartTime = undefined;\n        }\n\n        this.userTurnCommitted = false;\n      };\n\n    // cancel any existing EOU task\n    this.bounceEOUTask?.cancel();\n    // copy the values before awaiting (the values can change)\n    this.bounceEOUTask = Task.from(\n      bounceEOUTask(this.lastSpeakingTime, this.lastFinalTranscriptTime, this.speechStartTime),\n    );\n\n    this.bounceEOUTask.result\n      .then(() => {\n        this.logger.debug('EOU detection task completed');\n      })\n      .catch((err: unknown) => {\n        if (err instanceof Error && err.message.includes('This operation was aborted')) {\n          // ignore aborted errors\n          return;\n        }\n        this.logger.error(err, 'Error in EOU detection task:');\n      });\n  }\n\n  private async createSttTask(stt: STTNode | undefined, signal: AbortSignal) {\n    if (!stt) return;\n\n    this.logger.debug('createSttTask: create stt stream from stt node');\n\n    const sttStream = await stt(this.sttInputStream, {});\n\n    if (signal.aborted || sttStream === null) return;\n\n    if (sttStream instanceof ReadableStream) {\n      const reader = sttStream.getReader();\n\n      signal.addEventListener('abort', async () => {\n        try {\n          reader.releaseLock();\n          await sttStream?.cancel();\n        } catch (e) {\n          this.logger.debug('createSttTask: error during abort handler:', e);\n        }\n      });\n\n      try {\n        while (true) {\n          if (signal.aborted) break;\n\n          const { done, value: ev } = await reader.read();\n          if (done) break;\n\n          if (typeof ev === 'string') {\n            throw new Error('STT node must yield SpeechEvent');\n          } else {\n            await this.onSTTEvent(ev);\n          }\n        }\n      } catch (e) {\n        if (isStreamReaderReleaseError(e)) {\n          return;\n        }\n        this.logger.error({ error: e }, 'createSttTask: error reading sttStream');\n      } finally {\n        reader.releaseLock();\n        try {\n          await sttStream.cancel();\n        } catch (e) {\n          this.logger.debug(\n            'createSttTask: error cancelling sttStream (may already be cancelled):',\n            e,\n          );\n        }\n      }\n    }\n  }\n\n  private async createVadTask(vad: VAD | undefined, signal: AbortSignal) {\n    if (!vad) return;\n\n    const vadStream = vad.stream();\n    vadStream.updateInputStream(this.vadInputStream);\n\n    const abortHandler = () => {\n      vadStream.detachInputStream();\n      vadStream.close();\n      signal.removeEventListener('abort', abortHandler);\n    };\n    signal.addEventListener('abort', abortHandler);\n\n    try {\n      for await (const ev of vadStream) {\n        if (signal.aborted) break;\n\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.logger.debug('VAD task: START_OF_SPEECH');\n            this.hooks.onStartOfSpeech(ev);\n            this.speaking = true;\n\n            // Capture sample rate from the first VAD event if not already set\n            if (ev.frames.length > 0 && ev.frames[0]) {\n              this.sampleRate = ev.frames[0].sampleRate;\n            }\n\n            this.bounceEOUTask?.cancel();\n            break;\n          case VADEventType.INFERENCE_DONE:\n            this.hooks.onVADInferenceDone(ev);\n            // for metrics, get the \"earliest\" signal of speech as possible\n            if (ev.rawAccumulatedSpeech > 0.0) {\n              this.lastSpeakingTime = Date.now();\n\n              if (this.speechStartTime === undefined) {\n                this.speechStartTime = Date.now();\n              }\n            }\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.logger.debug('VAD task: END_OF_SPEECH');\n            this.hooks.onEndOfSpeech(ev);\n\n            // when VAD fires END_OF_SPEECH, it already waited for the silence_duration\n            this.speaking = false;\n\n            if (\n              this.vadBaseTurnDetection ||\n              (this.turnDetectionMode === 'stt' && this.userTurnCommitted)\n            ) {\n              const chatCtx = this.hooks.retrieveChatCtx();\n              this.runEOUDetection(chatCtx);\n            }\n            break;\n        }\n      }\n    } catch (e) {\n      this.logger.error(e, 'Error in VAD task');\n    } finally {\n      this.logger.debug('VAD task closed');\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputAudioStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  clearUserTurn() {\n    this.audioTranscript = '';\n    this.audioInterimTranscript = '';\n    this.audioPreflightTranscript = '';\n    this.finalTranscriptConfidence = [];\n    this.userTurnCommitted = false;\n\n    this.sttTask?.cancelAndWait().finally(() => {\n      this.sttTask = Task.from(({ signal }) => this.createSttTask(this.stt, signal));\n      this.sttTask.result.catch((err) => {\n        this.logger.error(`Error running STT task: ${err}`);\n      });\n    });\n  }\n\n  commitUserTurn(audioDetached: boolean) {\n    const commitUserTurnTask =\n      (delayDuration: number = 500) =>\n      async (controller: AbortController) => {\n        if (Date.now() - this.lastFinalTranscriptTime > delayDuration) {\n          // flush the stt by pushing silence\n          if (audioDetached && this.sampleRate !== undefined) {\n            const numSamples = Math.floor(this.sampleRate * 0.5);\n            const silence = new Int16Array(numSamples * 2);\n            const silenceFrame = new AudioFrame(silence, this.sampleRate, 1, numSamples);\n            this.silenceAudioWriter.write(silenceFrame);\n          }\n\n          // wait for the final transcript to be available\n          await delay(delayDuration, { signal: controller.signal });\n        }\n\n        if (this.audioInterimTranscript) {\n          // append interim transcript in case the final transcript is not ready\n          this.audioTranscript = `${this.audioTranscript} ${this.audioInterimTranscript}`.trim();\n        }\n        this.audioInterimTranscript = '';\n\n        const chatCtx = this.hooks.retrieveChatCtx();\n        this.logger.debug('running EOU detection on commitUserTurn');\n        this.runEOUDetection(chatCtx);\n        this.userTurnCommitted = true;\n      };\n\n    // cancel any existing commit user turn task\n    this.commitUserTurnTask?.cancel();\n    this.commitUserTurnTask = Task.from(commitUserTurnTask());\n\n    this.commitUserTurnTask.result\n      .then(() => {\n        this.logger.debug('User turn committed');\n      })\n      .catch((err: unknown) => {\n        this.logger.error(err, 'Error in user turn commit task:');\n      });\n  }\n\n  async close() {\n    this.detachInputAudioStream();\n    await this.commitUserTurnTask?.cancelAndWait();\n    await this.sttTask?.cancelAndWait();\n    await this.vadTask?.cancelAndWait();\n    await this.bounceEOUTask?.cancelAndWait();\n  }\n\n  private get vadBaseTurnDetection() {\n    return ['vad', undefined].includes(this.turnDetectionMode);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA2B;AAE3B,iBAA+B;AAC/B,0BAAiC;AACjC,iBAAoB;AACpB,6BAAmE;AACnE,gCAAkC;AAClC,oCAAqC;AACrC,iBAAkD;AAClD,mBAA4B;AAC5B,iBAAsD;AA8C/C,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA,aAAS,gBAAI;AAAA,EACb,0BAA0B;AAAA,EAC1B,kBAAkB;AAAA,EAClB,yBAAyB;AAAA,EACzB,2BAA2B;AAAA,EAC3B,4BAAsC,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA,oBAAoB;AAAA,EACpB,WAAW;AAAA,EACX;AAAA,EAEA;AAAA,EACA;AAAA,EACA,wBAAwB,IAAI,4CAA8B;AAAA,EAC1D;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,MAA+B;AACzC,SAAK,QAAQ,KAAK;AAClB,SAAK,MAAM,KAAK;AAChB,SAAK,MAAM,KAAK;AAChB,SAAK,eAAe,KAAK;AACzB,SAAK,oBAAoB,KAAK;AAC9B,SAAK,sBAAsB,KAAK;AAChC,SAAK,sBAAsB,KAAK;AAChC,SAAK,eAAe;AAEpB,SAAK,sBAAsB,IAAI,8CAAmC;AAClE,UAAM,CAAC,gBAAgB,cAAc,IAAI,KAAK,oBAAoB,OAAO,IAAI;AAC7E,SAAK,iBAAiB;AACtB,SAAK,qBAAiB,oDAAqB,gBAAgB,KAAK,sBAAsB,QAAQ;AAC9F,SAAK,qBAAqB,KAAK,sBAAsB,SAAS,UAAU;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,oBAA4B;AAC9B,QAAI,KAAK,wBAAwB;AAC/B,aAAO,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,IACvE;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,SAAK,UAAU,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAED,SAAK,UAAU,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,SAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,WAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,IACpD,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,WAAW,IAAiB;AApI5C;AAqII,QACE,KAAK,sBAAsB,YAC3B,KAAK,sBACJ,KAAK,kBAAkB,UACtB,KAAK,cAAc,QACnB,GAAG,QAAQ,2BAAgB,qBAC7B;AAGA,WAAK,OAAO;AAAA,QACV;AAAA,UACE,mBAAmB,KAAK;AAAA,UACxB,cAAa,UAAK,kBAAL,mBAAoB;AAAA,UACjC,QAAQ,GAAG;AAAA,UACX,mBAAmB,KAAK;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AACA;AAAA,IACF;AAEA,YAAQ,GAAG,MAAM;AAAA,MACf,KAAK,2BAAgB;AACnB,aAAK,MAAM,kBAAkB,EAAE;AAC/B,cAAM,cAAa,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AACzC,cAAM,eAAa,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,eAAc;AACvD,aAAK,gBAAe,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AAE1C,YAAI,CAAC,YAAY;AAEf;AAAA,QACF;AAEA,aAAK,OAAO;AAAA,UACV;AAAA,YACE,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,UACjB;AAAA,UACA;AAAA,QACF;AAEA,aAAK,0BAA0B,KAAK,IAAI;AACxC,aAAK,mBAAmB,IAAI,UAAU;AACtC,aAAK,kBAAkB,KAAK,gBAAgB,UAAU;AACtD,aAAK,0BAA0B,KAAK,UAAU;AAC9C,cAAM,oBAAoB,KAAK,oBAAoB,KAAK;AACxD,aAAK,yBAAyB;AAC9B,aAAK,2BAA2B;AAEhC,YAAI,CAAC,KAAK,OAAO,KAAK,qBAAqB,QAAW;AAMpD,eAAK,mBAAmB,KAAK,IAAI;AAAA,QACnC;AAEA,YAAI,KAAK,wBAAwB,KAAK,mBAAmB;AACvD,cAAI,mBAAmB;AACrB,iBAAK,OAAO;AAAA,cACV,EAAE,YAAY,KAAK,gBAAgB;AAAA,cACnC;AAAA,YACF;AACA,iBAAK,MAAM,uBAAuB;AAAA,cAChC,eAAe,KAAK;AAAA,cACpB,sBACE,KAAK,0BAA0B,SAAS,IACpC,KAAK,0BAA0B,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IACxD,KAAK,0BAA0B,SAC/B;AAAA,YACR,CAAC;AAAA,UACH;AAEA,cAAI,CAAC,KAAK,UAAU;AAClB,kBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,iBAAK,OAAO,MAAM,+CAA+C;AACjE,iBAAK,gBAAgB,OAAO;AAAA,UAC9B;AAAA,QACF;AACA;AAAA,MACF,KAAK,2BAAgB;AACnB,aAAK,MAAM,oBAAoB,EAAE;AACjC,cAAM,wBAAsB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,SAAQ;AAC1D,cAAM,wBAAsB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,eAAc;AAChE,cAAM,qBAAoB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB;AAEhD,cAAM,gCAAgC;AACtC,YACE,CAAC,KAAK,gBACL,qBAAqB,oBAAoB,SAAS,+BACnD;AACA,eAAK,eAAe;AAAA,QACtB;AAEA,YAAI,CAAC,qBAAqB;AACxB;AAAA,QACF;AAEA,aAAK,OAAO;AAAA,UACV;AAAA,YACE,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,UACjB;AAAA,UACA;AAAA,QACF;AAGA,aAAK,0BAA0B,KAAK,IAAI;AAExC,aAAK,2BACH,GAAG,KAAK,eAAe,IAAI,mBAAmB,GAAG,UAAU;AAC7D,aAAK,yBAAyB;AAE9B,YAAI,CAAC,KAAK,OAAO,KAAK,qBAAqB,QAAW;AAEpD,eAAK,mBAAmB,KAAK,IAAI;AAAA,QACnC;AAEA,YAAI,KAAK,sBAAsB,YAAY,KAAK,mBAAmB;AACjE,gBAAM,iBAAiB,CAAC,GAAG,KAAK,2BAA2B,mBAAmB;AAC9E,eAAK,OAAO;AAAA,YACV;AAAA,cACE,YACE,KAAK,yBAAyB,SAAS,MACnC,KAAK,yBAAyB,MAAM,GAAG,GAAG,IAAI,QAC9C,KAAK;AAAA,YACb;AAAA,YACA;AAAA,UACF;AACA,eAAK,MAAM,uBAAuB;AAAA,YAChC,eAAe,KAAK;AAAA,YACpB,sBACE,eAAe,SAAS,IACpB,eAAe,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,eAAe,SAC3D;AAAA,UACR,CAAC;AAAA,QACH;AACA;AAAA,MACF,KAAK,2BAAgB;AACnB,aAAK,OAAO,MAAM,EAAE,aAAY,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,KAAK,GAAG,oBAAoB;AAClF,aAAK,MAAM,oBAAoB,EAAE;AACjC,aAAK,2BAAyB,cAAG,iBAAH,mBAAkB,OAAlB,mBAAsB,SAAQ;AAC5D;AAAA,MACF,KAAK,2BAAgB;AACnB,YAAI,KAAK,sBAAsB,MAAO;AACtC,aAAK,MAAM,gBAAgB;AAAA,UACzB,MAAM,wBAAa;AAAA,UACnB,cAAc;AAAA,UACd,WAAW,KAAK,IAAI;AAAA,UACpB,gBAAgB;AAAA,UAChB,iBAAiB;AAAA,UACjB,QAAQ,CAAC;AAAA,UACT,aAAa;AAAA,UACb,mBAAmB;AAAA,UACnB,UAAU;AAAA,UACV,uBAAuB;AAAA,UACvB,sBAAsB;AAAA,QACxB,CAAC;AACD,aAAK,WAAW;AAChB,aAAK,mBAAmB,KAAK,IAAI;AAEjC,mBAAK,kBAAL,mBAAoB;AACpB;AAAA,MACF,KAAK,2BAAgB;AACnB,YAAI,KAAK,sBAAsB,MAAO;AACtC,aAAK,MAAM,cAAc;AAAA,UACvB,MAAM,wBAAa;AAAA,UACnB,cAAc;AAAA,UACd,WAAW,KAAK,IAAI;AAAA,UACpB,gBAAgB;AAAA,UAChB,iBAAiB;AAAA,UACjB,QAAQ,CAAC;AAAA,UACT,aAAa;AAAA,UACb,mBAAmB;AAAA,UACnB,UAAU;AAAA,UACV,uBAAuB;AAAA,UACvB,sBAAsB;AAAA,QACxB,CAAC;AACD,aAAK,WAAW;AAChB,aAAK,oBAAoB;AACzB,aAAK,mBAAmB,KAAK,IAAI;AAEjC,YAAI,CAAC,KAAK,UAAU;AAClB,gBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,eAAK,OAAO,MAAM,4CAA4C;AAC9D,eAAK,gBAAgB,OAAO;AAAA,QAC9B;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,gBAAgB,SAAsB;AApUhD;AAqUI,SAAK,OAAO;AAAA,MACV;AAAA,QACE,KAAK,KAAK;AAAA,QACV,iBAAiB,KAAK;AAAA,QACtB,mBAAmB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,CAAC,KAAK,mBAAmB,KAAK,sBAAsB,UAAU;AAE5E,WAAK,OAAO,MAAM,wBAAwB;AAC1C;AAAA,IACF;AAEA,cAAU,QAAQ,KAAK;AACvB,YAAQ,WAAW,EAAE,MAAM,QAAQ,SAAS,KAAK,gBAAgB,CAAC;AAElE,UAAM;AAAA;AAAA,MAEJ,KAAK,mBAAmB,KAAK,sBAAsB,WAAW,KAAK,eAAe;AAAA;AAEpF,UAAM,gBACJ,CACE,kBACA,yBACA,oBAEF,OAAO,eAAgC;AACrC,UAAI,mBAAmB,KAAK;AAE5B,UAAI,cAAc;AAChB,aAAK,OAAO,MAAM,6BAA6B;AAC/C,YAAI,CAAC,aAAa,iBAAiB,KAAK,YAAY,GAAG;AACrD,eAAK,OAAO,MAAM,2CAA2C,KAAK,YAAY,EAAE;AAAA,QAClF,OAAO;AACL,gBAAM,uBAAuB,MAAM,aAAa,iBAAiB,OAAO;AACxE,eAAK,OAAO;AAAA,YACV,EAAE,sBAAsB,UAAU,KAAK,aAAa;AAAA,YACpD;AAAA,UACF;AAEA,gBAAM,oBAAoB,MAAM,aAAa,kBAAkB,KAAK,YAAY;AAChF,eAAK,OAAO;AAAA,YACV;AAAA,cACE;AAAA,cACA;AAAA,cACA,UAAU,KAAK;AAAA,cACf,YAAY,KAAK;AAAA,YACnB;AAAA,YACA;AAAA,UACF;AAEA,cAAI,qBAAqB,uBAAuB,mBAAmB;AACjE,+BAAmB,KAAK;AAAA,UAC1B;AAAA,QACF;AAAA,MACF;AAEA,UAAI,aAAa;AACjB,UAAI,qBAAqB,QAAW;AAClC,sBAAc,mBAAmB,KAAK,IAAI;AAAA,MAC5C;AAEA,UAAI,aAAa,GAAG;AAElB,kBAAM,oBAAM,KAAK,IAAI,YAAY,CAAC,GAAG,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,MACpE;AAEA,WAAK,OAAO,MAAM,EAAE,YAAY,KAAK,gBAAgB,GAAG,kBAAkB;AAE1E,YAAM,gBACJ,KAAK,0BAA0B,SAAS,IACpC,KAAK,0BAA0B,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IACxD,KAAK,0BAA0B,SAC/B;AAEN,UAAI;AACJ,UAAI;AACJ,UAAI;AACJ,UAAI;AAIJ,UACE,4BAA4B,KAC5B,qBAAqB,UACrB,oBAAoB,QACpB;AACA,4BAAoB;AACpB,4BAAoB;AACpB,6BAAqB,KAAK,IAAI,0BAA0B,kBAAkB,CAAC;AAC3E,8BAAsB,KAAK,IAAI,IAAI;AAAA,MACrC;AAEA,YAAM,YAAY,MAAM,KAAK,MAAM,YAAY;AAAA,QAC7C,eAAe,KAAK;AAAA,QACpB,sBAAsB;AAAA,QACtB,oBAAoB,sBAAsB;AAAA,QAC1C,qBAAqB,uBAAuB;AAAA,QAC5C;AAAA,QACA;AAAA,MACF,CAAC;AAED,UAAI,WAAW;AAEb,aAAK,kBAAkB;AACvB,aAAK,4BAA4B,CAAC;AAClC,aAAK,mBAAmB;AACxB,aAAK,0BAA0B;AAC/B,aAAK,kBAAkB;AAAA,MACzB;AAEA,WAAK,oBAAoB;AAAA,IAC3B;AAGF,eAAK,kBAAL,mBAAoB;AAEpB,SAAK,gBAAgB,kBAAK;AAAA,MACxB,cAAc,KAAK,kBAAkB,KAAK,yBAAyB,KAAK,eAAe;AAAA,IACzF;AAEA,SAAK,cAAc,OAChB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,8BAA8B;AAAA,IAClD,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,UAAI,eAAe,SAAS,IAAI,QAAQ,SAAS,4BAA4B,GAAG;AAE9E;AAAA,MACF;AACA,WAAK,OAAO,MAAM,KAAK,8BAA8B;AAAA,IACvD,CAAC;AAAA,EACL;AAAA,EAEA,MAAc,cAAc,KAA0B,QAAqB;AACzE,QAAI,CAAC,IAAK;AAEV,SAAK,OAAO,MAAM,gDAAgD;AAElE,UAAM,YAAY,MAAM,IAAI,KAAK,gBAAgB,CAAC,CAAC;AAEnD,QAAI,OAAO,WAAW,cAAc,KAAM;AAE1C,QAAI,qBAAqB,2BAAgB;AACvC,YAAM,SAAS,UAAU,UAAU;AAEnC,aAAO,iBAAiB,SAAS,YAAY;AAC3C,YAAI;AACF,iBAAO,YAAY;AACnB,iBAAM,uCAAW;AAAA,QACnB,SAAS,GAAG;AACV,eAAK,OAAO,MAAM,8CAA8C,CAAC;AAAA,QACnE;AAAA,MACF,CAAC;AAED,UAAI;AACF,eAAO,MAAM;AACX,cAAI,OAAO,QAAS;AAEpB,gBAAM,EAAE,MAAM,OAAO,GAAG,IAAI,MAAM,OAAO,KAAK;AAC9C,cAAI,KAAM;AAEV,cAAI,OAAO,OAAO,UAAU;AAC1B,kBAAM,IAAI,MAAM,iCAAiC;AAAA,UACnD,OAAO;AACL,kBAAM,KAAK,WAAW,EAAE;AAAA,UAC1B;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,gBAAI,mDAA2B,CAAC,GAAG;AACjC;AAAA,QACF;AACA,aAAK,OAAO,MAAM,EAAE,OAAO,EAAE,GAAG,wCAAwC;AAAA,MAC1E,UAAE;AACA,eAAO,YAAY;AACnB,YAAI;AACF,gBAAM,UAAU,OAAO;AAAA,QACzB,SAAS,GAAG;AACV,eAAK,OAAO;AAAA,YACV;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,KAAsB,QAAqB;AAlgBzE;AAmgBI,QAAI,CAAC,IAAK;AAEV,UAAM,YAAY,IAAI,OAAO;AAC7B,cAAU,kBAAkB,KAAK,cAAc;AAE/C,UAAM,eAAe,MAAM;AACzB,gBAAU,kBAAkB;AAC5B,gBAAU,MAAM;AAChB,aAAO,oBAAoB,SAAS,YAAY;AAAA,IAClD;AACA,WAAO,iBAAiB,SAAS,YAAY;AAE7C,QAAI;AACF,uBAAiB,MAAM,WAAW;AAChC,YAAI,OAAO,QAAS;AAEpB,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,wBAAa;AAChB,iBAAK,OAAO,MAAM,2BAA2B;AAC7C,iBAAK,MAAM,gBAAgB,EAAE;AAC7B,iBAAK,WAAW;AAGhB,gBAAI,GAAG,OAAO,SAAS,KAAK,GAAG,OAAO,CAAC,GAAG;AACxC,mBAAK,aAAa,GAAG,OAAO,CAAC,EAAE;AAAA,YACjC;AAEA,uBAAK,kBAAL,mBAAoB;AACpB;AAAA,UACF,KAAK,wBAAa;AAChB,iBAAK,MAAM,mBAAmB,EAAE;AAEhC,gBAAI,GAAG,uBAAuB,GAAK;AACjC,mBAAK,mBAAmB,KAAK,IAAI;AAEjC,kBAAI,KAAK,oBAAoB,QAAW;AACtC,qBAAK,kBAAkB,KAAK,IAAI;AAAA,cAClC;AAAA,YACF;AACA;AAAA,UACF,KAAK,wBAAa;AAChB,iBAAK,OAAO,MAAM,yBAAyB;AAC3C,iBAAK,MAAM,cAAc,EAAE;AAG3B,iBAAK,WAAW;AAEhB,gBACE,KAAK,wBACJ,KAAK,sBAAsB,SAAS,KAAK,mBAC1C;AACA,oBAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,mBAAK,gBAAgB,OAAO;AAAA,YAC9B;AACA;AAAA,QACJ;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,GAAG,mBAAmB;AAAA,IAC1C,UAAE;AACA,WAAK,OAAO,MAAM,iBAAiB;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,oBAAoB,aAAyC;AAC3D,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,yBAAyB;AACvB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA,EAEA,gBAAgB;AA3kBlB;AA4kBI,SAAK,kBAAkB;AACvB,SAAK,yBAAyB;AAC9B,SAAK,2BAA2B;AAChC,SAAK,4BAA4B,CAAC;AAClC,SAAK,oBAAoB;AAEzB,eAAK,YAAL,mBAAc,gBAAgB,QAAQ,MAAM;AAC1C,WAAK,UAAU,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,MAAM,CAAC;AAC7E,WAAK,QAAQ,OAAO,MAAM,CAAC,QAAQ;AACjC,aAAK,OAAO,MAAM,2BAA2B,GAAG,EAAE;AAAA,MACpD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,eAAe,eAAwB;AA1lBzC;AA2lBI,UAAM,qBACJ,CAAC,gBAAwB,QACzB,OAAO,eAAgC;AACrC,UAAI,KAAK,IAAI,IAAI,KAAK,0BAA0B,eAAe;AAE7D,YAAI,iBAAiB,KAAK,eAAe,QAAW;AAClD,gBAAM,aAAa,KAAK,MAAM,KAAK,aAAa,GAAG;AACnD,gBAAM,UAAU,IAAI,WAAW,aAAa,CAAC;AAC7C,gBAAM,eAAe,IAAI,2BAAW,SAAS,KAAK,YAAY,GAAG,UAAU;AAC3E,eAAK,mBAAmB,MAAM,YAAY;AAAA,QAC5C;AAGA,kBAAM,oBAAM,eAAe,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,MAC1D;AAEA,UAAI,KAAK,wBAAwB;AAE/B,aAAK,kBAAkB,GAAG,KAAK,eAAe,IAAI,KAAK,sBAAsB,GAAG,KAAK;AAAA,MACvF;AACA,WAAK,yBAAyB;AAE9B,YAAM,UAAU,KAAK,MAAM,gBAAgB;AAC3C,WAAK,OAAO,MAAM,yCAAyC;AAC3D,WAAK,gBAAgB,OAAO;AAC5B,WAAK,oBAAoB;AAAA,IAC3B;AAGF,eAAK,uBAAL,mBAAyB;AACzB,SAAK,qBAAqB,kBAAK,KAAK,mBAAmB,CAAC;AAExD,SAAK,mBAAmB,OACrB,KAAK,MAAM;AACV,WAAK,OAAO,MAAM,qBAAqB;AAAA,IACzC,CAAC,EACA,MAAM,CAAC,QAAiB;AACvB,WAAK,OAAO,MAAM,KAAK,iCAAiC;AAAA,IAC1D,CAAC;AAAA,EACL;AAAA,EAEA,MAAM,QAAQ;AApoBhB;AAqoBI,SAAK,uBAAuB;AAC5B,YAAM,UAAK,uBAAL,mBAAyB;AAC/B,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,YAAL,mBAAc;AACpB,YAAM,UAAK,kBAAL,mBAAoB;AAAA,EAC5B;AAAA,EAEA,IAAY,uBAAuB;AACjC,WAAO,CAAC,OAAO,MAAS,EAAE,SAAS,KAAK,iBAAiB;AAAA,EAC3D;AACF;","names":[]}

package/dist/voice/audio_recognition.d.cts

@@ -8,8 +8,15 @@ import type { TurnDetectionMode } from './agent_session.js';
 import type { STTNode } from './io.js';
 export interface EndOfTurnInfo {
     newTranscript: string;
+    transcriptConfidence: number;
     transcriptionDelay: number;
     endOfUtteranceDelay: number;
+    startedSpeakingAt: number | undefined;
+    stoppedSpeakingAt: number | undefined;
+}
+export interface PreemptiveGenerationInfo {
+    newTranscript: string;
+    transcriptConfidence: number;
 }
 export interface RecognitionHooks {
     onStartOfSpeech: (ev: VADEvent) => void;
@@ -18,6 +25,7 @@ export interface RecognitionHooks {
     onInterimTranscript: (ev: SpeechEvent) => void;
     onFinalTranscript: (ev: SpeechEvent) => void;
     onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;
+    onPreemptiveGeneration: (info: PreemptiveGenerationInfo) => void;
     retrieveChatCtx: () => ChatContext;
 }
 export interface _TurnDetector {
@@ -48,7 +56,10 @@ export declare class AudioRecognition {
     private lastFinalTranscriptTime;
     private audioTranscript;
     private audioInterimTranscript;
+    private audioPreflightTranscript;
+    private finalTranscriptConfidence;
     private lastSpeakingTime;
+    private speechStartTime;
     private userTurnCommitted;
     private speaking;
     private sampleRate?;

package/dist/voice/audio_recognition.d.ts

@@ -8,8 +8,15 @@ import type { TurnDetectionMode } from './agent_session.js';
 import type { STTNode } from './io.js';
 export interface EndOfTurnInfo {
     newTranscript: string;
+    transcriptConfidence: number;
     transcriptionDelay: number;
     endOfUtteranceDelay: number;
+    startedSpeakingAt: number | undefined;
+    stoppedSpeakingAt: number | undefined;
+}
+export interface PreemptiveGenerationInfo {
+    newTranscript: string;
+    transcriptConfidence: number;
 }
 export interface RecognitionHooks {
     onStartOfSpeech: (ev: VADEvent) => void;
@@ -18,6 +25,7 @@ export interface RecognitionHooks {
     onInterimTranscript: (ev: SpeechEvent) => void;
     onFinalTranscript: (ev: SpeechEvent) => void;
     onEndOfTurn: (info: EndOfTurnInfo) => Promise<boolean>;
+    onPreemptiveGeneration: (info: PreemptiveGenerationInfo) => void;
     retrieveChatCtx: () => ChatContext;
 }
 export interface _TurnDetector {
@@ -48,7 +56,10 @@ export declare class AudioRecognition {
     private lastFinalTranscriptTime;
     private audioTranscript;
     private audioInterimTranscript;
+    private audioPreflightTranscript;
+    private finalTranscriptConfidence;
     private lastSpeakingTime;
+    private speechStartTime;
     private userTurnCommitted;
     private speaking;
     private sampleRate?;