@livekit/agents 1.1.0-dev.0 → 1.2.0

package/dist/voice/agent_activity.d.cts

@@ -74,6 +74,7 @@ export declare class AgentActivity implements RecognitionHooks {
     get allowInterruptions(): boolean;
     get useTtsAlignedTranscript(): boolean;
     get turnDetection(): TurnDetectionMode | undefined;
+    get turnHandling(): Partial<import("./turn_config/turn_handling.js").TurnHandlingOptions>;
     get toolCtx(): ToolContext;
     /** @internal */
     get inputStartedAt(): number | undefined;
@@ -147,6 +148,7 @@ export declare class AgentActivity implements RecognitionHooks {
     close(): Promise<void>;
     private resolveInterruptionDetector;
     private restoreInterruptionByAudioActivity;
+    private fallbackToVadInterruption;
     private _closeSessionResources;
 }
 export {};

package/dist/voice/agent_activity.d.ts

@@ -74,6 +74,7 @@ export declare class AgentActivity implements RecognitionHooks {
     get allowInterruptions(): boolean;
     get useTtsAlignedTranscript(): boolean;
     get turnDetection(): TurnDetectionMode | undefined;
+    get turnHandling(): Partial<import("./turn_config/turn_handling.js").TurnHandlingOptions>;
     get toolCtx(): ToolContext;
     /** @internal */
     get inputStartedAt(): number | undefined;
@@ -147,6 +148,7 @@ export declare class AgentActivity implements RecognitionHooks {
     close(): Promise<void>;
     private resolveInterruptionDetector;
     private restoreInterruptionByAudioActivity;
+    private fallbackToVadInterruption;
     private _closeSessionResources;
 }
 export {};

package/dist/voice/agent_activity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";;AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAIpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAmB,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAsB,MAAM,wBAAwB,CAAC;AAC3F,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EAEjB,MAAM,iBAAiB,CAAC;AAczB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAQvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACtB,MAAM,wBAAwB,CAAC;AAsBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,eAAO,MAAM,oBAAoB,kCAAyC,CAAC;AAC3E,eAAO,MAAM,cAAc,gCAAuC,CAAC;AAEnE,UAAU,WAAW;IACnB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC;CACd;AAaD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IAEzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,iBAAiB,CAAQ;IACjC,OAAO,CAAC,kBAAkB,CAAmB;IAC7C,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,aAAa,CAAC,CAAS;IAG/B,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IACrD,OAAO,CAAC,oBAAoB,CAAC,CAA+B;IAC5D,OAAO,CAAC,8BAA8B,CAAU;IAChD,OAAO,CAAC,oCAAoC,CAAU;IACtD,OAAO,CAAC,2CAA2C,CAAU;IAE7D,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CACb;IAE/B,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CACb;IAEhC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CACb;IAEhC,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAEJ;IAEvD,OAAO,CAAC,QAAQ,CAAC,YAAY,CACV;IAEnB,OAAO,CAAC,QAAQ,CAAC,+BAA+B,CAE9C;IAEF,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAK7C;IAEF,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAIlC;IAEF,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,YAAY,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,WAAW,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,sBAAsB,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAExB,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IAwG9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;YASf,aAAa;IA4H3B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,OAAO,CAAC,cAAc;IAWtB,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,uBAAuB,IAAI,OAAO,CAGrC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,gBAAgB;IAChB,IAAI,cAAc,uBAEjB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlD,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAapD,aAAa,CAAC,EACZ,UAAU,EACV,aAAa,GACd,EAAE;QACD,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,aAAa,CAAC,EAAE,iBAAiB,CAAC;KACnC,GAAG,IAAI;IA0BR,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAgC/D,gBAAgB,IAAI,IAAI;IAUxB,cAAc,CACZ,OAAO,GAAE;QACP,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB;IAaR,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAuDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IA0BxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAqBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAoCrD,eAAe,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAiBnC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAgBjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAWtC,OAAO,CAAC,wBAAwB;IAsDhC,cAAc,CAAC,EAAE,EAAE,sBAAsB;IAQzC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAqB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IA+BxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA2C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IAiDlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA6CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAyCtB,OAAO,CAAC,0BAA0B;IAiClC,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAkHhB,SAAS,CAAC,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IA6B1D,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAoJjB,OAAO;IA6JrB,OAAO,CAAC,sBAAsB,CA4b5B;IAEF,OAAO,CAAC,iBAAiB,CA6BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;IA8azC,OAAO,CAAC,4BAA4B;YA+CtB,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;YAiBR,oBAAoB;IAelC,OAAO,CAAC,qBAAqB;IAOvB,KAAK,CAAC,OAAO,GAAE;QAAE,YAAY,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA0BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAwB5B,OAAO,CAAC,2BAA2B;IA2CnC,OAAO,CAAC,kCAAkC;YAI5B,sBAAsB;CAwCrC"}
+{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";;AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAIpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAmB,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAsB,MAAM,wBAAwB,CAAC;AAC3F,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EAEjB,MAAM,iBAAiB,CAAC;AAczB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA+C,MAAM,aAAa,CAAC;AACxF,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAQvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACtB,MAAM,wBAAwB,CAAC;AAsBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,eAAO,MAAM,oBAAoB,kCAAyC,CAAC;AAC3E,eAAO,MAAM,cAAc,gCAAuC,CAAC;AAEnE,UAAU,WAAW;IACnB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC;CACd;AAaD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IAEzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,iBAAiB,CAAQ;IACjC,OAAO,CAAC,kBAAkB,CAAmB;IAC7C,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,aAAa,CAAC,CAAS;IAG/B,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IACrD,OAAO,CAAC,oBAAoB,CAAC,CAA+B;IAC5D,OAAO,CAAC,8BAA8B,CAAU;IAChD,OAAO,CAAC,oCAAoC,CAAU;IACtD,OAAO,CAAC,2CAA2C,CAAU;IAE7D,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CACb;IAE/B,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CACb;IAEhC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CACb;IAEhC,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAEJ;IAEvD,OAAO,CAAC,QAAQ,CAAC,YAAY,CACV;IAEnB,OAAO,CAAC,QAAQ,CAAC,+BAA+B,CAE9C;IAEF,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAM7C;IAEF,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAWlC;IAEF,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,YAAY,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,WAAW,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,sBAAsB,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAExB,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IAwG9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;YASf,aAAa;IAgI3B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,OAAO,CAAC,cAAc;IAWtB,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAKhC;IAED,IAAI,uBAAuB,IAAI,OAAO,CAGrC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAEjD;IAED,IAAI,YAAY,0EAEf;IAgBD,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,gBAAgB;IAChB,IAAI,cAAc,uBAEjB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlD,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAapD,aAAa,CAAC,EACZ,UAAU,EACV,aAAa,GACd,EAAE;QACD,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,aAAa,CAAC,EAAE,iBAAiB,CAAC;KACnC,GAAG,IAAI;IA0BR,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAqC/D,gBAAgB,IAAI,IAAI;IAUxB,cAAc,CACZ,OAAO,GAAE;QACP,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB;IAaR,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAuDf,OAAO,CAAC,kBAAkB,CAuBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IA0BxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAqBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAoCrD,eAAe,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAoBnC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAmBjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAatC,OAAO,CAAC,wBAAwB;IAsDhC,cAAc,CAAC,EAAE,EAAE,sBAAsB;IAQzC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAqB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IA+BxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA2C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IAiDlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA8CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAmDtB,OAAO,CAAC,0BAA0B;IAiClC,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAkHhB,SAAS,CAAC,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IA8C1D,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAoJjB,OAAO;IA6JrB,OAAO,CAAC,sBAAsB,CA0b5B;IAEF,OAAO,CAAC,iBAAiB,CA6BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;IA8azC,OAAO,CAAC,4BAA4B;YA+CtB,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;YAiBR,oBAAoB;IAelC,OAAO,CAAC,qBAAqB;IAOvB,KAAK,CAAC,OAAO,GAAE;QAAE,YAAY,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA0BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B5B,OAAO,CAAC,2BAA2B;IA6DnC,OAAO,CAAC,kCAAkC;IAI1C,OAAO,CAAC,yBAAyB;YAwBnB,sBAAsB;CAwCrC"}

package/dist/voice/agent_activity.js

@@ -17,7 +17,7 @@ import { STT } from "../stt/stt.js";
 import { recordRealtimeMetrics, traceTypes, tracer } from "../telemetry/index.js";
 import { splitWords } from "../tokenize/basic/word.js";
 import { TTS } from "../tts/tts.js";
-import { Future, Task, cancelAndWait, waitFor } from "../utils.js";
+import { Future, Task, cancelAndWait, isDevMode, isHosted, waitFor } from "../utils.js";
 import { VAD } from "../vad.js";
 import {
   StopResponse,
@@ -85,9 +85,10 @@ class AgentActivity {
   onRealtimeInputAudioTranscriptionCompleted = (ev) => this.onInputAudioTranscriptionCompleted(ev);
   onModelError = (ev) => this.onError(ev);
   onInterruptionOverlappingSpeech = (ev) => {
-    this.agentSession.emit(AgentSessionEventTypes.UserOverlappingSpeech, ev);
+    this.agentSession.emit(AgentSessionEventTypes.OverlappingSpeech, ev);
   };
   onInterruptionMetricsCollected = (ev) => {
+    this.agentSession._usageCollector.collect(ev);
     this.agentSession.emit(
       AgentSessionEventTypes.MetricsCollected,
       createMetricsCollectedEvent({ metrics: ev })
@@ -96,6 +97,11 @@ class AgentActivity {
   onInterruptionError = (ev) => {
     const errorEvent = createErrorEvent(ev, this.interruptionDetector);
     this.agentSession.emit(AgentSessionEventTypes.Error, errorEvent);
+    if (!ev.recoverable) {
+      this.agentSession._onError(ev);
+      this.fallbackToVadInterruption();
+      return;
+    }
     this.agentSession._onError(ev);
   };
   /** @internal */
@@ -183,7 +189,7 @@ class AgentActivity {
     }
   }
   async _startSession(options) {
-    var _a;
+    var _a, _b, _c, _d, _e;
     const { spanName, runOnEnter } = options;
     const startSpan = tracer.startSpan({
       name: spanName,
@@ -258,10 +264,10 @@ class AgentActivity {
       turnDetector: typeof this.turnDetection === "string" ? void 0 : this.turnDetection,
       turnDetectionMode: this.turnDetectionMode,
       interruptionDetection: this.interruptionDetector,
-      minEndpointingDelay: this.agentSession.options.turnHandling.endpointing.minDelay,
-      maxEndpointingDelay: this.agentSession.options.turnHandling.endpointing.maxDelay,
+      minEndpointingDelay: ((_b = (_a = this.agent.turnHandling) == null ? void 0 : _a.endpointing) == null ? void 0 : _b.minDelay) ?? this.agentSession.sessionOptions.turnHandling.endpointing.minDelay,
+      maxEndpointingDelay: ((_d = (_c = this.agent.turnHandling) == null ? void 0 : _c.endpointing) == null ? void 0 : _d.maxDelay) ?? this.agentSession.sessionOptions.turnHandling.endpointing.maxDelay,
       rootSpanContext: this.agentSession.rootSpanContext,
-      sttModel: (_a = this.stt) == null ? void 0 : _a.label,
+      sttModel: (_e = this.stt) == null ? void 0 : _e.label,
       sttProvider: this.getSttProvider(),
       getLinkedParticipant: () => {
         var _a2;
@@ -321,15 +327,31 @@ class AgentActivity {
     return this.realtimeSession;
   }
   get allowInterruptions() {
-    var _a;
-    return ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.mode) !== false;
+    var _a, _b;
+    return ((_b = (_a = this.agent.turnHandling) == null ? void 0 : _a.interruption) == null ? void 0 : _b.enabled) ?? this.agentSession.sessionOptions.turnHandling.interruption.enabled;
   }
   get useTtsAlignedTranscript() {
     return this.agent.useTtsAlignedTranscript ?? this.agentSession.useTtsAlignedTranscript;
   }
   get turnDetection() {
-    return this.agentSession.turnDetection;
-  }
+    var _a;
+    return ((_a = this.agent.turnHandling) == null ? void 0 : _a.turnDetection) ?? this.agentSession.turnDetection;
+  }
+  get turnHandling() {
+    return this.agent.turnHandling ?? this.agentSession.sessionOptions.turnHandling;
+  }
+  // get minEndpointingDelay(): number {
+  //   return (
+  //     this.agent.turnHandling?.endpointing?.minDelay ??
+  //     this.agentSession.sessionOptions.turnHandling.endpointing.minDelay
+  //   );
+  // }
+  // get maxEndpointingDelay(): number {
+  //   return (
+  //     this.agent.turnHandling?.endpointing?.maxDelay ??
+  //     this.agentSession.sessionOptions.turnHandling.endpointing.maxDelay
+  //   );
+  // }
   get toolCtx() {
     return this.agent.toolCtx;
   }
@@ -395,12 +417,18 @@ class AgentActivity {
       }
     });
     this.audioStreamId = this.audioStream.addInputStream(audioStream);
-    const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.pipeThrough(aecWarmupAudioFilter).tee();
-    if (this.realtimeSession) {
+    if (this.realtimeSession && this.audioRecognition) {
+      const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.pipeThrough(aecWarmupAudioFilter).tee();
       this.realtimeSession.setInputAudioStream(realtimeAudioStream);
-    }
-    if (this.audioRecognition) {
       this.audioRecognition.setInputAudioStream(recognitionAudioStream);
+    } else if (this.realtimeSession) {
+      this.realtimeSession.setInputAudioStream(
+        this.audioStream.stream.pipeThrough(aecWarmupAudioFilter)
+      );
+    } else if (this.audioRecognition) {
+      this.audioRecognition.setInputAudioStream(
+        this.audioStream.stream.pipeThrough(aecWarmupAudioFilter)
+      );
     }
   }
   detachAudioInput() {
@@ -475,6 +503,7 @@ class AgentActivity {
         this.realtimeSpans.delete(ev.requestId);
       }
     }
+    this.agentSession._usageCollector.collect(ev);
     this.agentSession.emit(
       AgentSessionEventTypes.MetricsCollected,
       createMetricsCollectedEvent({ metrics: ev })
@@ -587,7 +616,10 @@ class AgentActivity {
     if (ev) {
       speechStartTime = speechStartTime - ev.speechDuration - ev.inferenceDuration;
     }
-    this.agentSession._updateUserState("speaking", speechStartTime);
+    this.agentSession._updateUserState("speaking", {
+      lastSpeakingTime: speechStartTime,
+      otelContext: otelContext.active()
+    });
     if (this.isInterruptionDetectionEnabled && this.audioRecognition) {
       this.audioRecognition.onStartOfOverlapSpeech(
         ev.speechDuration,
@@ -607,14 +639,17 @@ class AgentActivity {
         this.agentSession._userSpeakingSpan
       );
     }
-    this.agentSession._updateUserState("listening", speechEndTime);
+    this.agentSession._updateUserState("listening", {
+      lastSpeakingTime: speechEndTime,
+      otelContext: otelContext.active()
+    });
   }
   onVADInferenceDone(ev) {
     var _a;
     if (this.turnDetection === "manual" || this.turnDetection === "realtime_llm") {
       return;
     }
-    if (ev.speechDuration >= ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.minDuration)) {
+    if (ev.speechDuration >= ((_a = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _a.minDuration)) {
       this.interruptByAudioActivity();
     }
   }
@@ -629,11 +664,11 @@ class AgentActivity {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
       return;
     }
-    if (this.stt && ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0 && this.audioRecognition) {
+    if (this.stt && ((_a = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0 && this.audioRecognition) {
       const text = this.audioRecognition.currentTranscript;
       const normalizedText = text ?? "";
       const wordCount = splitWords(normalizedText, true).length;
-      if (wordCount < ((_b = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
+      if (wordCount < ((_b = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
         return;
       }
     }
@@ -651,7 +686,7 @@ class AgentActivity {
     this.restoreInterruptionByAudioActivity();
     this.interruptByAudioActivity();
     if (this.audioRecognition) {
-      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.timestamp);
+      this.audioRecognition.onEndOfAgentSpeech(ev.overlapStartedAt || ev.detectedAt);
     }
   }
   onInterimTranscript(ev) {
@@ -689,7 +724,7 @@ class AgentActivity {
     }
   }
   onPreemptiveGeneration(info) {
-    if (!this.agentSession.options.preemptiveGeneration || this.schedulingPaused || this._currentSpeech !== void 0 && !this._currentSpeech.interrupted || !(this.llm instanceof LLM)) {
+    if (!this.agentSession.sessionOptions.preemptiveGeneration || this.schedulingPaused || this._currentSpeech !== void 0 && !this._currentSpeech.interrupted || !(this.llm instanceof LLM)) {
       return;
     }
     this.cancelPreemptiveGeneration();
@@ -770,14 +805,14 @@ class AgentActivity {
       );
       return true;
     }
-    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && ((_a = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0) {
+    if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && ((_a = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _a.minWords) > 0) {
       const wordCount = splitWords(info.newTranscript, true).length;
-      if (wordCount < ((_b = this.agentSession.options.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
+      if (wordCount < ((_b = this.agentSession.sessionOptions.turnHandling.interruption) == null ? void 0 : _b.minWords)) {
         this.cancelPreemptiveGeneration();
         this.logger.info(
           {
             wordCount,
-            minInterruptionWords: this.agentSession.options.turnHandling.interruption.minWords
+            minInterruptionWords: this.agentSession.sessionOptions.turnHandling.interruption.minWords
           },
           "skipping user input, word count below minimum interruption threshold"
         );
@@ -811,9 +846,12 @@ class AgentActivity {
           throw new Error("Speech queue is empty");
         }
         const speechHandle = heapItem[2];
+        if (speechHandle.interrupted || speechHandle.done()) {
+          continue;
+        }
         this._currentSpeech = speechHandle;
         speechHandle._authorizeGeneration();
-        await speechHandle._waitForGeneration();
+        await speechHandle.waitIfNotInterrupted([speechHandle._waitForGeneration()]);
         this._currentSpeech = void 0;
       }
       const toWait = this.getDrainPendingSpeechTasks();
@@ -953,7 +991,16 @@ ${instructions}`;
       speech.interrupt(force);
     }
     (_a = this.realtimeSession) == null ? void 0 : _a.interrupt();
-    if (currentSpeech === void 0) {
+    if (force) {
+      for (const task of this.speechTasks) {
+        task.cancel();
+      }
+      if (currentSpeech && !currentSpeech.done()) {
+        currentSpeech._markDone();
+      }
+      this.speechQueue.clear();
+      future.resolve();
+    } else if (currentSpeech === void 0) {
       future.resolve();
     } else {
       currentSpeech.addDoneCallback(() => {
@@ -1400,9 +1447,7 @@ ${instructions}`;
         audioOutput.clearBuffer();
       }
       replyAbortController.abort();
-      await Promise.allSettled(
-        tasks.map((task) => task.cancelAndWait(AgentActivity.REPLY_TASK_CANCEL_TIMEOUT))
-      );
+      await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
       let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
       if (audioOutput) {
         const playbackEv = await audioOutput.waitForPlayout();
@@ -1483,7 +1528,7 @@ ${instructions}`;
     speechHandle._markGenerationDone();
     await executeToolsTask.result;
     if (toolOutput.output.length === 0) return;
-    const { maxToolSteps } = this.agentSession.options;
+    const { maxToolSteps } = this.agentSession.sessionOptions;
     if (speechHandle.numSteps >= maxToolSteps) {
       this.logger.warn(
         { speech_id: speechHandle.id, max_tool_steps: maxToolSteps },
@@ -1834,7 +1879,7 @@ ${instructions}`;
     if (toolOutput.output.length === 0) {
       return;
     }
-    const { maxToolSteps } = this.agentSession.options;
+    const { maxToolSteps } = this.agentSession.sessionOptions;
     if (speechHandle.numSteps >= maxToolSteps) {
       this.logger.warn(
         { speech_id: speechHandle.id, max_tool_steps: maxToolSteps },
@@ -2053,15 +2098,16 @@ ${instructions}`;
     const unlock = await this.lock.lock();
     try {
       this.cancelPreemptiveGeneration();
+      await cancelAndWait(Array.from(this.speechTasks), AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
+      if (this._currentSpeech && !this._currentSpeech.done()) {
+        this._currentSpeech._markDone();
+      }
       await this._closeSessionResources();
       if (this._mainTask) {
         await this._mainTask.cancelAndWait();
       }
       if (this.interruptionDetector) {
-        this.interruptionDetector.off(
-          "user_overlapping_speech",
-          this.onInterruptionOverlappingSpeech
-        );
+        this.interruptionDetector.off("overlapping_speech", this.onInterruptionOverlappingSpeech);
         this.interruptionDetector.off("metrics_collected", this.onInterruptionMetricsCollected);
         this.interruptionDetector.off("error", this.onInterruptionError);
       }
@@ -2071,21 +2117,33 @@ ${instructions}`;
     }
   }
   resolveInterruptionDetector() {
-    const interruptionDetection = this.agent.interruptionDetection ?? this.agentSession.interruptionDetection;
+    var _a, _b;
+    const agentInterruptionDetection = (_b = (_a = this.agent.turnHandling) == null ? void 0 : _a.interruption) == null ? void 0 : _b.mode;
+    const sessionInterruptionDetection = this.agentSession.interruptionDetection;
     if (!(this.stt && this.stt.capabilities.alignedTranscript && this.stt.capabilities.streaming && this.vad && this.turnDetection !== "manual" && this.turnDetection !== "realtime_llm" && !(this.llm instanceof RealtimeModel))) {
-      if (interruptionDetection === "adaptive") {
+      if (agentInterruptionDetection === "adaptive" || sessionInterruptionDetection === "adaptive") {
         this.logger.warn(
           "interruptionDetection is provided, but it's not compatible with the current configuration and will be disabled"
         );
-        return void 0;
       }
+      return void 0;
+    }
+    if (!this.allowInterruptions) {
+      return void 0;
+    }
+    if (agentInterruptionDetection === "vad") {
+      return void 0;
     }
-    if (interruptionDetection !== void 0 && interruptionDetection === false || interruptionDetection === "vad") {
+    if (sessionInterruptionDetection === "vad") {
+      return void 0;
+    }
+    if (agentInterruptionDetection === void 0 && sessionInterruptionDetection === void 0 && !isHosted() && !isDevMode()) {
+      this.logger.info("adaptive interruption is disabled by default in production mode");
       return void 0;
     }
     try {
       const detector = new AdaptiveInterruptionDetector();
-      detector.on("user_overlapping_speech", this.onInterruptionOverlappingSpeech);
+      detector.on("overlapping_speech", this.onInterruptionOverlappingSpeech);
       detector.on("metrics_collected", this.onInterruptionMetricsCollected);
       detector.on("error", this.onInterruptionError);
       return detector;
@@ -2097,6 +2155,25 @@ ${instructions}`;
   restoreInterruptionByAudioActivity() {
     this.isInterruptionByAudioActivityEnabled = this.isDefaultInterruptionByAudioActivityEnabled;
   }
+  fallbackToVadInterruption() {
+    if (!this.isInterruptionDetectionEnabled) return;
+    this.isInterruptionDetectionEnabled = false;
+    this.restoreInterruptionByAudioActivity();
+    if (this.interruptionDetector) {
+      this.interruptionDetector.off("overlapping_speech", this.onInterruptionOverlappingSpeech);
+      this.interruptionDetector.off("metrics_collected", this.onInterruptionMetricsCollected);
+      this.interruptionDetector.off("error", this.onInterruptionError);
+      this.interruptionDetector = void 0;
+    }
+    if (this.audioRecognition) {
+      this.audioRecognition.disableInterruptionDetection().catch((err) => {
+        this.logger.warn({ err }, "error while disabling interruption detection");
+      });
+    }
+    this.logger.warn(
+      "adaptive interruption disabled due to unrecoverable error, falling back to VAD-based interruption"
+    );
+  }
   async _closeSessionResources() {
     var _a, _b, _c;
     if (this.llm instanceof LLM) {