@livekit/agents 1.1.0-dev.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +2 -0
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +2 -0
- package/dist/cli.js.map +1 -1
- package/dist/constants.cjs +3 -0
- package/dist/constants.cjs.map +1 -1
- package/dist/constants.d.cts +1 -0
- package/dist/constants.d.ts +1 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +2 -0
- package/dist/constants.js.map +1 -1
- package/dist/cpu.cjs +189 -0
- package/dist/cpu.cjs.map +1 -0
- package/dist/cpu.d.cts +24 -0
- package/dist/cpu.d.ts +24 -0
- package/dist/cpu.d.ts.map +1 -0
- package/dist/cpu.js +152 -0
- package/dist/cpu.js.map +1 -0
- package/dist/cpu.test.cjs +227 -0
- package/dist/cpu.test.cjs.map +1 -0
- package/dist/cpu.test.js +204 -0
- package/dist/cpu.test.js.map +1 -0
- package/dist/index.cjs +12 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -13
- package/dist/index.d.ts +13 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -10
- package/dist/index.js.map +1 -1
- package/dist/inference/interruption/defaults.cjs +1 -1
- package/dist/inference/interruption/defaults.cjs.map +1 -1
- package/dist/inference/interruption/defaults.d.cts +1 -1
- package/dist/inference/interruption/defaults.d.ts +1 -1
- package/dist/inference/interruption/defaults.d.ts.map +1 -1
- package/dist/inference/interruption/defaults.js +1 -1
- package/dist/inference/interruption/defaults.js.map +1 -1
- package/dist/inference/interruption/http_transport.cjs +44 -28
- package/dist/inference/interruption/http_transport.cjs.map +1 -1
- package/dist/inference/interruption/http_transport.d.ts.map +1 -1
- package/dist/inference/interruption/http_transport.js +45 -29
- package/dist/inference/interruption/http_transport.js.map +1 -1
- package/dist/inference/interruption/interruption_detector.cjs +22 -5
- package/dist/inference/interruption/interruption_detector.cjs.map +1 -1
- package/dist/inference/interruption/interruption_detector.d.cts +2 -2
- package/dist/inference/interruption/interruption_detector.d.ts +2 -2
- package/dist/inference/interruption/interruption_detector.d.ts.map +1 -1
- package/dist/inference/interruption/interruption_detector.js +22 -5
- package/dist/inference/interruption/interruption_detector.js.map +1 -1
- package/dist/inference/interruption/interruption_stream.cjs +4 -4
- package/dist/inference/interruption/interruption_stream.cjs.map +1 -1
- package/dist/inference/interruption/interruption_stream.js +4 -4
- package/dist/inference/interruption/interruption_stream.js.map +1 -1
- package/dist/inference/interruption/types.cjs.map +1 -1
- package/dist/inference/interruption/types.d.cts +2 -2
- package/dist/inference/interruption/types.d.ts +2 -2
- package/dist/inference/interruption/types.d.ts.map +1 -1
- package/dist/inference/interruption/ws_transport.cjs +60 -47
- package/dist/inference/interruption/ws_transport.cjs.map +1 -1
- package/dist/inference/interruption/ws_transport.d.ts.map +1 -1
- package/dist/inference/interruption/ws_transport.js +60 -47
- package/dist/inference/interruption/ws_transport.js.map +1 -1
- package/dist/inference/llm.cjs.map +1 -1
- package/dist/inference/llm.d.cts +1 -1
- package/dist/inference/llm.d.ts +1 -1
- package/dist/inference/llm.d.ts.map +1 -1
- package/dist/inference/llm.js.map +1 -1
- package/dist/inference/stt.cjs +20 -12
- package/dist/inference/stt.cjs.map +1 -1
- package/dist/inference/stt.d.cts +3 -2
- package/dist/inference/stt.d.ts +3 -2
- package/dist/inference/stt.d.ts.map +1 -1
- package/dist/inference/stt.js +20 -12
- package/dist/inference/stt.js.map +1 -1
- package/dist/inference/stt.test.cjs +14 -0
- package/dist/inference/stt.test.cjs.map +1 -1
- package/dist/inference/stt.test.js +14 -0
- package/dist/inference/stt.test.js.map +1 -1
- package/dist/inference/tts.cjs +13 -4
- package/dist/inference/tts.cjs.map +1 -1
- package/dist/inference/tts.d.cts +8 -1
- package/dist/inference/tts.d.ts +8 -1
- package/dist/inference/tts.d.ts.map +1 -1
- package/dist/inference/tts.js +13 -4
- package/dist/inference/tts.js.map +1 -1
- package/dist/inference/tts.test.cjs +10 -0
- package/dist/inference/tts.test.cjs.map +1 -1
- package/dist/inference/tts.test.js +10 -0
- package/dist/inference/tts.test.js.map +1 -1
- package/dist/ipc/job_proc_lazy_main.cjs +41 -23
- package/dist/ipc/job_proc_lazy_main.cjs.map +1 -1
- package/dist/ipc/job_proc_lazy_main.js +41 -23
- package/dist/ipc/job_proc_lazy_main.js.map +1 -1
- package/dist/job.cjs +1 -1
- package/dist/job.cjs.map +1 -1
- package/dist/job.js +1 -1
- package/dist/job.js.map +1 -1
- package/dist/language.cjs +394 -0
- package/dist/language.cjs.map +1 -0
- package/dist/language.d.cts +15 -0
- package/dist/language.d.ts +15 -0
- package/dist/language.d.ts.map +1 -0
- package/dist/language.js +363 -0
- package/dist/language.js.map +1 -0
- package/dist/language.test.cjs +43 -0
- package/dist/language.test.cjs.map +1 -0
- package/dist/language.test.js +49 -0
- package/dist/language.test.js.map +1 -0
- package/dist/llm/index.cjs +2 -0
- package/dist/llm/index.cjs.map +1 -1
- package/dist/llm/index.d.cts +1 -1
- package/dist/llm/index.d.ts +1 -1
- package/dist/llm/index.d.ts.map +1 -1
- package/dist/llm/index.js +2 -0
- package/dist/llm/index.js.map +1 -1
- package/dist/stream/deferred_stream.cjs +6 -2
- package/dist/stream/deferred_stream.cjs.map +1 -1
- package/dist/stream/deferred_stream.d.ts.map +1 -1
- package/dist/stream/deferred_stream.js +6 -2
- package/dist/stream/deferred_stream.js.map +1 -1
- package/dist/stt/stt.cjs.map +1 -1
- package/dist/stt/stt.d.cts +2 -1
- package/dist/stt/stt.d.ts +2 -1
- package/dist/stt/stt.d.ts.map +1 -1
- package/dist/stt/stt.js.map +1 -1
- package/dist/utils.cjs +15 -0
- package/dist/utils.cjs.map +1 -1
- package/dist/utils.d.cts +8 -0
- package/dist/utils.d.ts +8 -0
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +13 -0
- package/dist/utils.js.map +1 -1
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/voice/agent.cjs +14 -17
- package/dist/voice/agent.cjs.map +1 -1
- package/dist/voice/agent.d.cts +10 -11
- package/dist/voice/agent.d.ts +10 -11
- package/dist/voice/agent.d.ts.map +1 -1
- package/dist/voice/agent.js +15 -18
- package/dist/voice/agent.js.map +1 -1
- package/dist/voice/agent.test.cjs +194 -0
- package/dist/voice/agent.test.cjs.map +1 -1
- package/dist/voice/agent.test.js +195 -1
- package/dist/voice/agent.test.js.map +1 -1
- package/dist/voice/agent_activity.cjs +116 -39
- package/dist/voice/agent_activity.cjs.map +1 -1
- package/dist/voice/agent_activity.d.cts +2 -0
- package/dist/voice/agent_activity.d.ts +2 -0
- package/dist/voice/agent_activity.d.ts.map +1 -1
- package/dist/voice/agent_activity.js +117 -40
- package/dist/voice/agent_activity.js.map +1 -1
- package/dist/voice/agent_activity.test.cjs +135 -0
- package/dist/voice/agent_activity.test.cjs.map +1 -0
- package/dist/voice/agent_activity.test.js +134 -0
- package/dist/voice/agent_activity.test.js.map +1 -0
- package/dist/voice/agent_session.cjs +38 -38
- package/dist/voice/agent_session.cjs.map +1 -1
- package/dist/voice/agent_session.d.cts +65 -56
- package/dist/voice/agent_session.d.ts +65 -56
- package/dist/voice/agent_session.d.ts.map +1 -1
- package/dist/voice/agent_session.js +37 -37
- package/dist/voice/agent_session.js.map +1 -1
- package/dist/voice/audio_recognition.cjs +106 -52
- package/dist/voice/audio_recognition.cjs.map +1 -1
- package/dist/voice/audio_recognition.d.cts +4 -2
- package/dist/voice/audio_recognition.d.ts +4 -2
- package/dist/voice/audio_recognition.d.ts.map +1 -1
- package/dist/voice/audio_recognition.js +106 -52
- package/dist/voice/audio_recognition.js.map +1 -1
- package/dist/voice/audio_recognition_span.test.cjs +84 -22
- package/dist/voice/audio_recognition_span.test.cjs.map +1 -1
- package/dist/voice/audio_recognition_span.test.js +90 -23
- package/dist/voice/audio_recognition_span.test.js.map +1 -1
- package/dist/voice/events.cjs +1 -1
- package/dist/voice/events.cjs.map +1 -1
- package/dist/voice/events.d.cts +4 -3
- package/dist/voice/events.d.ts +4 -3
- package/dist/voice/events.d.ts.map +1 -1
- package/dist/voice/events.js +1 -1
- package/dist/voice/events.js.map +1 -1
- package/dist/voice/index.cjs +9 -1
- package/dist/voice/index.cjs.map +1 -1
- package/dist/voice/index.d.cts +1 -1
- package/dist/voice/index.d.ts +1 -1
- package/dist/voice/index.d.ts.map +1 -1
- package/dist/voice/index.js +10 -1
- package/dist/voice/index.js.map +1 -1
- package/dist/voice/remote_session.cjs +922 -0
- package/dist/voice/remote_session.cjs.map +1 -0
- package/dist/voice/remote_session.d.cts +108 -0
- package/dist/voice/remote_session.d.ts +108 -0
- package/dist/voice/remote_session.d.ts.map +1 -0
- package/dist/voice/remote_session.js +887 -0
- package/dist/voice/remote_session.js.map +1 -0
- package/dist/voice/report.cjs +11 -10
- package/dist/voice/report.cjs.map +1 -1
- package/dist/voice/report.d.cts +5 -3
- package/dist/voice/report.d.ts +5 -3
- package/dist/voice/report.d.ts.map +1 -1
- package/dist/voice/report.js +11 -10
- package/dist/voice/report.js.map +1 -1
- package/dist/voice/report.test.cjs +15 -0
- package/dist/voice/report.test.cjs.map +1 -1
- package/dist/voice/report.test.js +15 -0
- package/dist/voice/report.test.js.map +1 -1
- package/dist/voice/room_io/room_io.cjs +39 -0
- package/dist/voice/room_io/room_io.cjs.map +1 -1
- package/dist/voice/room_io/room_io.d.cts +3 -1
- package/dist/voice/room_io/room_io.d.ts +3 -1
- package/dist/voice/room_io/room_io.d.ts.map +1 -1
- package/dist/voice/room_io/room_io.js +40 -1
- package/dist/voice/room_io/room_io.js.map +1 -1
- package/dist/voice/turn_config/interruption.cjs.map +1 -1
- package/dist/voice/turn_config/interruption.d.cts +1 -1
- package/dist/voice/turn_config/interruption.d.ts +1 -1
- package/dist/voice/turn_config/interruption.d.ts.map +1 -1
- package/dist/voice/turn_config/interruption.js.map +1 -1
- package/dist/voice/turn_config/utils.cjs +95 -35
- package/dist/voice/turn_config/utils.cjs.map +1 -1
- package/dist/voice/turn_config/utils.d.cts +17 -5
- package/dist/voice/turn_config/utils.d.ts +17 -5
- package/dist/voice/turn_config/utils.d.ts.map +1 -1
- package/dist/voice/turn_config/utils.js +93 -35
- package/dist/voice/turn_config/utils.js.map +1 -1
- package/dist/voice/turn_config/utils.test.cjs +83 -41
- package/dist/voice/turn_config/utils.test.cjs.map +1 -1
- package/dist/voice/turn_config/utils.test.js +84 -42
- package/dist/voice/turn_config/utils.test.js.map +1 -1
- package/dist/worker.cjs +6 -29
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +6 -19
- package/dist/worker.js.map +1 -1
- package/package.json +3 -2
- package/src/cli.ts +2 -0
- package/src/constants.ts +1 -0
- package/src/cpu.test.ts +239 -0
- package/src/cpu.ts +173 -0
- package/src/index.ts +13 -15
- package/src/inference/interruption/defaults.ts +1 -1
- package/src/inference/interruption/http_transport.ts +49 -30
- package/src/inference/interruption/interruption_detector.ts +22 -6
- package/src/inference/interruption/interruption_stream.ts +4 -4
- package/src/inference/interruption/types.ts +2 -2
- package/src/inference/interruption/ws_transport.ts +63 -59
- package/src/inference/llm.ts +3 -1
- package/src/inference/stt.test.ts +17 -0
- package/src/inference/stt.ts +22 -14
- package/src/inference/tts.test.ts +12 -0
- package/src/inference/tts.ts +22 -6
- package/src/ipc/job_proc_lazy_main.ts +44 -24
- package/src/job.ts +1 -1
- package/src/language.test.ts +62 -0
- package/src/language.ts +380 -0
- package/src/llm/index.ts +2 -0
- package/src/stream/deferred_stream.ts +5 -1
- package/src/stt/stt.ts +2 -1
- package/src/utils.ts +20 -0
- package/src/voice/agent.test.ts +208 -1
- package/src/voice/agent.ts +21 -22
- package/src/voice/agent_activity.test.ts +194 -0
- package/src/voice/agent_activity.ts +161 -43
- package/src/voice/agent_session.ts +103 -92
- package/src/voice/audio_recognition.ts +124 -61
- package/src/voice/audio_recognition_span.test.ts +115 -35
- package/src/voice/events.ts +4 -3
- package/src/voice/index.ts +10 -1
- package/src/voice/remote_session.ts +1083 -0
- package/src/voice/report.test.ts +22 -3
- package/src/voice/report.ts +31 -14
- package/src/voice/room_io/room_io.ts +52 -2
- package/src/voice/turn_config/interruption.ts +1 -1
- package/src/voice/turn_config/utils.test.ts +91 -43
- package/src/voice/turn_config/utils.ts +120 -56
- package/src/worker.ts +34 -50
- package/dist/voice/client_events.cjs +0 -554
- package/dist/voice/client_events.cjs.map +0 -1
- package/dist/voice/client_events.d.cts +0 -195
- package/dist/voice/client_events.d.ts +0 -195
- package/dist/voice/client_events.d.ts.map +0 -1
- package/dist/voice/client_events.js +0 -548
- package/dist/voice/client_events.js.map +0 -1
- package/dist/voice/wire_format.cjs +0 -798
- package/dist/voice/wire_format.cjs.map +0 -1
- package/dist/voice/wire_format.d.cts +0 -5503
- package/dist/voice/wire_format.d.ts +0 -5503
- package/dist/voice/wire_format.d.ts.map +0 -1
- package/dist/voice/wire_format.js +0 -728
- package/dist/voice/wire_format.js.map +0 -1
- package/src/voice/client_events.ts +0 -838
- package/src/voice/wire_format.ts +0 -827
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Span } from '@opentelemetry/api';
|
|
2
2
|
export interface OverlappingSpeechEvent {
|
|
3
|
-
type: '
|
|
4
|
-
|
|
3
|
+
type: 'overlapping_speech';
|
|
4
|
+
detectedAt: number;
|
|
5
5
|
isInterruption: boolean;
|
|
6
6
|
totalDurationInS: number;
|
|
7
7
|
predictionDurationInS: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/types.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE/C,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/types.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE/C,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,oBAAoB,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,0BAA0B,EAAE,MAAM,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;CACjB;AAID,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,sBAAsB,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,oBAAoB,CAAC;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,8FAA8F;IAC9F,cAAc,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,uGAAuG;IACvG,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAC5B,kBAAkB,GAClB,gBAAgB,GAChB,oBAAoB,GACpB,kBAAkB,GAClB,KAAK,CAAC"}
|
|
@@ -34,9 +34,9 @@ module.exports = __toCommonJS(ws_transport_exports);
|
|
|
34
34
|
var import_web = require("stream/web");
|
|
35
35
|
var import_ws = __toESM(require("ws"), 1);
|
|
36
36
|
var import_zod = require("zod");
|
|
37
|
+
var import_exceptions = require("../../_exceptions.cjs");
|
|
37
38
|
var import_log = require("../../log.cjs");
|
|
38
39
|
var import_utils = require("../utils.cjs");
|
|
39
|
-
var import_defaults = require("./defaults.cjs");
|
|
40
40
|
var import_interruption_cache_entry = require("./interruption_cache_entry.cjs");
|
|
41
41
|
const MSG_SESSION_CREATE = "session.create";
|
|
42
42
|
const MSG_SESSION_CLOSE = "session.close";
|
|
@@ -82,16 +82,32 @@ async function connectWebSocket(options) {
|
|
|
82
82
|
await new Promise((resolve, reject) => {
|
|
83
83
|
const timeout = setTimeout(() => {
|
|
84
84
|
ws.terminate();
|
|
85
|
-
reject(
|
|
85
|
+
reject(
|
|
86
|
+
new import_exceptions.APITimeoutError({
|
|
87
|
+
message: "WebSocket connection timeout",
|
|
88
|
+
options: { retryable: false }
|
|
89
|
+
})
|
|
90
|
+
);
|
|
86
91
|
}, options.timeout);
|
|
87
92
|
ws.once("open", () => {
|
|
88
93
|
clearTimeout(timeout);
|
|
89
94
|
resolve();
|
|
90
95
|
});
|
|
96
|
+
ws.once("unexpected-response", (_req, res) => {
|
|
97
|
+
clearTimeout(timeout);
|
|
98
|
+
ws.terminate();
|
|
99
|
+
const statusCode = res.statusCode ?? -1;
|
|
100
|
+
reject(
|
|
101
|
+
new import_exceptions.APIStatusError({
|
|
102
|
+
message: `WebSocket connection rejected with status ${statusCode}`,
|
|
103
|
+
options: { statusCode, retryable: false }
|
|
104
|
+
})
|
|
105
|
+
);
|
|
106
|
+
});
|
|
91
107
|
ws.once("error", (err) => {
|
|
92
108
|
clearTimeout(timeout);
|
|
93
109
|
ws.terminate();
|
|
94
|
-
reject(err);
|
|
110
|
+
reject(new import_exceptions.APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
|
|
95
111
|
});
|
|
96
112
|
});
|
|
97
113
|
return ws;
|
|
@@ -110,7 +126,9 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
110
126
|
}
|
|
111
127
|
});
|
|
112
128
|
socket.on("error", (err) => {
|
|
113
|
-
|
|
129
|
+
outputController == null ? void 0 : outputController.error(
|
|
130
|
+
new import_exceptions.APIConnectionError({ message: `WebSocket error: ${err.message}` })
|
|
131
|
+
);
|
|
114
132
|
});
|
|
115
133
|
socket.on("close", (code, reason) => {
|
|
116
134
|
logger.debug({ code, reason: reason.toString() }, "WebSocket closed");
|
|
@@ -118,37 +136,19 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
118
136
|
}
|
|
119
137
|
async function ensureConnection() {
|
|
120
138
|
if (ws && ws.readyState === import_ws.default.OPEN) return;
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
num_channels: 1,
|
|
132
|
-
threshold: options.threshold,
|
|
133
|
-
min_frames: options.minFrames,
|
|
134
|
-
encoding: "s16le"
|
|
135
|
-
}
|
|
136
|
-
});
|
|
137
|
-
ws.send(sessionCreateMsg);
|
|
138
|
-
return;
|
|
139
|
-
} catch (err) {
|
|
140
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
141
|
-
if (attempt < maxRetries) {
|
|
142
|
-
const delay = (0, import_defaults.intervalForRetry)(attempt);
|
|
143
|
-
logger.debug(
|
|
144
|
-
{ attempt, delay, err: lastError.message },
|
|
145
|
-
"WebSocket connection failed, retrying"
|
|
146
|
-
);
|
|
147
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
148
|
-
}
|
|
139
|
+
ws = await connectWebSocket(options);
|
|
140
|
+
setupMessageHandler(ws);
|
|
141
|
+
const sessionCreateMsg = JSON.stringify({
|
|
142
|
+
type: MSG_SESSION_CREATE,
|
|
143
|
+
settings: {
|
|
144
|
+
sample_rate: options.sampleRate,
|
|
145
|
+
num_channels: 1,
|
|
146
|
+
threshold: options.threshold,
|
|
147
|
+
min_frames: options.minFrames,
|
|
148
|
+
encoding: "s16le"
|
|
149
149
|
}
|
|
150
|
-
}
|
|
151
|
-
|
|
150
|
+
});
|
|
151
|
+
ws.send(sessionCreateMsg);
|
|
152
152
|
}
|
|
153
153
|
function handleMessage(message) {
|
|
154
154
|
const state = getState();
|
|
@@ -188,8 +188,8 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
188
188
|
"interruption detected"
|
|
189
189
|
);
|
|
190
190
|
const event = {
|
|
191
|
-
type: "
|
|
192
|
-
|
|
191
|
+
type: "overlapping_speech",
|
|
192
|
+
detectedAt: Date.now(),
|
|
193
193
|
isInterruption: true,
|
|
194
194
|
totalDurationInS: entry.totalDurationInS,
|
|
195
195
|
predictionDurationInS: entry.predictionDurationInS,
|
|
@@ -239,16 +239,17 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
239
239
|
break;
|
|
240
240
|
case MSG_ERROR:
|
|
241
241
|
outputController == null ? void 0 : outputController.error(
|
|
242
|
-
new
|
|
243
|
-
`LiveKit Adaptive Interruption error
|
|
244
|
-
|
|
242
|
+
new import_exceptions.APIStatusError({
|
|
243
|
+
message: `LiveKit Adaptive Interruption error: ${message.message}`,
|
|
244
|
+
options: { statusCode: message.code ?? -1 }
|
|
245
|
+
})
|
|
245
246
|
);
|
|
246
247
|
break;
|
|
247
248
|
}
|
|
248
249
|
}
|
|
249
250
|
function sendAudioData(audioSlice) {
|
|
250
251
|
if (!ws || ws.readyState !== import_ws.default.OPEN) {
|
|
251
|
-
throw new
|
|
252
|
+
throw new import_exceptions.APIConnectionError({ message: "WebSocket not connected" });
|
|
252
253
|
}
|
|
253
254
|
const state = getState();
|
|
254
255
|
const createdAt = Math.floor(performance.now());
|
|
@@ -272,12 +273,8 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
272
273
|
const combined = new Uint8Array(8 + audioBytes.length);
|
|
273
274
|
combined.set(new Uint8Array(header), 0);
|
|
274
275
|
combined.set(audioBytes, 8);
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
onRequestSent == null ? void 0 : onRequestSent();
|
|
278
|
-
} catch (e) {
|
|
279
|
-
logger.error(e, `failed to send audio via websocket`);
|
|
280
|
-
}
|
|
276
|
+
ws.send(combined);
|
|
277
|
+
onRequestSent == null ? void 0 : onRequestSent();
|
|
281
278
|
}
|
|
282
279
|
function close() {
|
|
283
280
|
if ((ws == null ? void 0 : ws.readyState) === import_ws.default.OPEN) {
|
|
@@ -307,10 +304,26 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
307
304
|
}
|
|
308
305
|
const state = getState();
|
|
309
306
|
if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;
|
|
307
|
+
if (options.timeout > 0) {
|
|
308
|
+
const now = performance.now();
|
|
309
|
+
for (const [, entry] of state.cache.entries()) {
|
|
310
|
+
if (entry.totalDurationInS !== 0) continue;
|
|
311
|
+
if (now - entry.createdAt > options.timeout) {
|
|
312
|
+
controller.error(
|
|
313
|
+
new import_exceptions.APIStatusError({
|
|
314
|
+
message: `interruption inference timed out after ${((now - entry.createdAt) / 1e3).toFixed(1)}s (ws)`,
|
|
315
|
+
options: { statusCode: 408, retryable: false }
|
|
316
|
+
})
|
|
317
|
+
);
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
break;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
310
323
|
try {
|
|
311
324
|
sendAudioData(chunk);
|
|
312
325
|
} catch (err) {
|
|
313
|
-
|
|
326
|
+
controller.error(err);
|
|
314
327
|
}
|
|
315
328
|
},
|
|
316
329
|
flush() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { intervalForRetry } from './defaults.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new Promise<void>((resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(new Error('WebSocket connection timeout'));\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(err);\n });\n });\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n logger.error({ err }, 'WebSocket error');\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<void> {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n const maxRetries = options.maxRetries ?? 3;\n let lastError: Error | null = null;\n\n for (let attempt = 0; attempt <= maxRetries; attempt++) {\n try {\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n // Send session.create message\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n return;\n } catch (err) {\n lastError = err instanceof Error ? err : new Error(String(err));\n if (attempt < maxRetries) {\n const delay = intervalForRetry(attempt);\n logger.debug(\n { attempt, delay, err: lastError.message },\n 'WebSocket connection failed, retrying',\n );\n await new Promise((resolve) => setTimeout(resolve, delay));\n }\n }\n }\n\n throw lastError ?? new Error('Failed to connect to WebSocket after retries');\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'user_overlapping_speech',\n timestamp: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new Error(\n `LiveKit Adaptive Interruption error${\n message.code !== undefined ? ` (${message.code})` : ''\n }: ${message.message}`,\n ),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new Error('WebSocket not connected');\n }\n\n const state = getState();\n // Use truncated timestamp consistently for both cache key and header\n // This ensures the server's response created_at matches our cache key\n const createdAt = Math.floor(performance.now());\n\n // Store the audio data in cache with truncated timestamp\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n // Create header: 8-byte little-endian uint64 timestamp (milliseconds as integer)\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n // Combine header and audio data\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n try {\n ws.send(combined);\n onRequestSent?.();\n } catch (e: unknown) {\n logger.error(e, `failed to send audio via websocket`);\n }\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection();\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n logger.error({ err }, 'Failed to send audio data over WebSocket');\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAAgC;AAChC,gBAAsB;AACtB,iBAAkB;AAClB,iBAAoB;AACpB,mBAAkC;AAClC,sBAAiC;AACjC,sCAAuC;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,aAAE,mBAAmB,QAAQ;AAAA,EACnD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,aAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,aAAE,OAAO;AAAA,IAClB,MAAM,aAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,aAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,UAAM,gCAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAAA,QAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb,aAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,IAClD,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,GAAG;AAAA,IACZ,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,aAAS,gBAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,aAAO,MAAM,EAAE,IAAI,GAAG,iBAAiB;AAAA,IACzC,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAAA,QAAU,KAAM;AAE5C,UAAM,aAAa,QAAQ,cAAc;AACzC,QAAI,YAA0B;AAE9B,aAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,UAAI;AACF,aAAK,MAAM,iBAAiB,OAAO;AACnC,4BAAoB,EAAE;AAGtB,cAAM,mBAAmB,KAAK,UAAU;AAAA,UACtC,MAAM;AAAA,UACN,UAAU;AAAA,YACR,aAAa,QAAQ;AAAA,YACrB,cAAc;AAAA,YACd,WAAW,QAAQ;AAAA,YACnB,YAAY,QAAQ;AAAA,YACpB,UAAU;AAAA,UACZ;AAAA,QACF,CAAC;AACD,WAAG,KAAK,gBAAgB;AACxB;AAAA,MACF,SAAS,KAAK;AACZ,oBAAY,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC9D,YAAI,UAAU,YAAY;AACxB,gBAAM,YAAQ,kCAAiB,OAAO;AACtC,iBAAO;AAAA,YACL,EAAE,SAAS,OAAO,KAAK,UAAU,QAAQ;AAAA,YACzC;AAAA,UACF;AACA,gBAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,IACF;AAEA,UAAM,aAAa,IAAI,MAAM,8CAA8C;AAAA,EAC7E;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,WAAW,KAAK,IAAI;AAAA,YACpB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI;AAAA,YACF,sCACE,QAAQ,SAAS,SAAY,KAAK,QAAQ,IAAI,MAAM,EACtD,KAAK,QAAQ,OAAO;AAAA,UACtB;AAAA;AAEF;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAAA,QAAU,MAAM;AAC3C,YAAM,IAAI,MAAM,yBAAyB;AAAA,IAC3C;AAEA,UAAM,QAAQ,SAAS;AAGvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAG9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uDAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAGA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAGjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,QAAI;AACF,SAAG,KAAK,QAAQ;AAChB;AAAA,IACF,SAAS,GAAY;AACnB,aAAO,MAAM,GAAG,oCAAoC;AAAA,IACtD;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAAA,QAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,iBAAO,MAAM,EAAE,IAAI,GAAG,0CAA0C;AAAA,QAClE;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":["WebSocket"]}
|
|
1
|
+
{"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new Promise<void>((resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(\n new APITimeoutError({\n message: 'WebSocket connection timeout',\n options: { retryable: false },\n }),\n );\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('unexpected-response', (_req, res) => {\n clearTimeout(timeout);\n ws.terminate();\n const statusCode = res.statusCode ?? -1;\n reject(\n new APIStatusError({\n message: `WebSocket connection rejected with status ${statusCode}`,\n options: { statusCode, retryable: false },\n }),\n );\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));\n });\n });\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n outputController?.error(\n new APIConnectionError({ message: `WebSocket error: ${err.message}` }),\n );\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<void> {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new APIStatusError({\n message: `LiveKit Adaptive Interruption error: ${message.message}`,\n options: { statusCode: message.code ?? -1 },\n }),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new APIConnectionError({ message: 'WebSocket not connected' });\n }\n\n const state = getState();\n const createdAt = Math.floor(performance.now());\n\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n ws.send(combined);\n onRequestSent?.();\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection();\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n if (options.timeout > 0) {\n const now = performance.now();\n for (const [, entry] of state.cache.entries()) {\n if (entry.totalDurationInS !== 0) continue;\n if (now - entry.createdAt > options.timeout) {\n controller.error(\n new APIStatusError({\n message: `interruption inference timed out after ${((now - entry.createdAt) / 1000).toFixed(1)}s (ws)`,\n options: { statusCode: 408, retryable: false },\n }),\n );\n return;\n }\n break;\n }\n }\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n controller.error(err);\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAAgC;AAChC,gBAAsB;AACtB,iBAAkB;AAClB,wBAAoE;AACpE,iBAAoB;AACpB,mBAAkC;AAClC,sCAAuC;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,aAAE,mBAAmB,QAAQ;AAAA,EACnD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,aAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,aAAE,OAAO;AAAA,IAClB,MAAM,aAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,aAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,UAAM,gCAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAAA,QAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb;AAAA,QACE,IAAI,kCAAgB;AAAA,UAClB,SAAS;AAAA,UACT,SAAS,EAAE,WAAW,MAAM;AAAA,QAC9B,CAAC;AAAA,MACH;AAAA,IACF,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,uBAAuB,CAAC,MAAM,QAAQ;AAC5C,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,YAAM,aAAa,IAAI,cAAc;AACrC;AAAA,QACE,IAAI,iCAAe;AAAA,UACjB,SAAS,6CAA6C,UAAU;AAAA,UAChE,SAAS,EAAE,YAAY,WAAW,MAAM;AAAA,QAC1C,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,IAAI,qCAAmB,EAAE,SAAS,+BAA+B,IAAI,OAAO,GAAG,CAAC,CAAC;AAAA,IAC1F,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,aAAS,gBAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,2DAAkB;AAAA,QAChB,IAAI,qCAAmB,EAAE,SAAS,oBAAoB,IAAI,OAAO,GAAG,CAAC;AAAA;AAAA,IAEzE,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAAA,QAAU,KAAM;AAE5C,SAAK,MAAM,iBAAiB,OAAO;AACnC,wBAAoB,EAAE;AAEtB,UAAM,mBAAmB,KAAK,UAAU;AAAA,MACtC,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa,QAAQ;AAAA,QACrB,cAAc;AAAA,QACd,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB,UAAU;AAAA,MACZ;AAAA,IACF,CAAC;AACD,OAAG,KAAK,gBAAgB;AAAA,EAC1B;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,YAAY,KAAK,IAAI;AAAA,YACrB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI,iCAAe;AAAA,YACjB,SAAS,wCAAwC,QAAQ,OAAO;AAAA,YAChE,SAAS,EAAE,YAAY,QAAQ,QAAQ,GAAG;AAAA,UAC5C,CAAC;AAAA;AAEH;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAAA,QAAU,MAAM;AAC3C,YAAM,IAAI,qCAAmB,EAAE,SAAS,0BAA0B,CAAC;AAAA,IACrE;AAEA,UAAM,QAAQ,SAAS;AACvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAE9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uDAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAEjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,OAAG,KAAK,QAAQ;AAChB;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAAA,QAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI,QAAQ,UAAU,GAAG;AACvB,gBAAM,MAAM,YAAY,IAAI;AAC5B,qBAAW,CAAC,EAAE,KAAK,KAAK,MAAM,MAAM,QAAQ,GAAG;AAC7C,gBAAI,MAAM,qBAAqB,EAAG;AAClC,gBAAI,MAAM,MAAM,YAAY,QAAQ,SAAS;AAC3C,yBAAW;AAAA,gBACT,IAAI,iCAAe;AAAA,kBACjB,SAAS,4CAA4C,MAAM,MAAM,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,kBAC9F,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,gBAC/C,CAAC;AAAA,cACH;AACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF;AAEA,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":["WebSocket"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ws_transport.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/ws_transport.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAM7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAW/C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CACrD;AA+
|
|
1
|
+
{"version":3,"file":"ws_transport.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/ws_transport.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAM7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAW/C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CACrD;AA+ED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,eAAe,CAAC,UAAU,GAAG,sBAAsB,EAAE,sBAAsB,CAAC,CAAC;IACxF,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,MAAM,gBAAgB,EAChC,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC,KAAK,IAAI,EACtD,sBAAsB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EAChE,aAAa,CAAC,EAAE,MAAM,IAAI,EAC1B,sBAAsB,CAAC,EAAE,MAAM,MAAM,GACpC,iBAAiB,CA8QnB"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { TransformStream } from "stream/web";
|
|
2
2
|
import WebSocket from "ws";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
+
import { APIConnectionError, APIStatusError, APITimeoutError } from "../../_exceptions.js";
|
|
4
5
|
import { log } from "../../log.js";
|
|
5
6
|
import { createAccessToken } from "../utils.js";
|
|
6
|
-
import { intervalForRetry } from "./defaults.js";
|
|
7
7
|
import { InterruptionCacheEntry } from "./interruption_cache_entry.js";
|
|
8
8
|
const MSG_SESSION_CREATE = "session.create";
|
|
9
9
|
const MSG_SESSION_CLOSE = "session.close";
|
|
@@ -49,16 +49,32 @@ async function connectWebSocket(options) {
|
|
|
49
49
|
await new Promise((resolve, reject) => {
|
|
50
50
|
const timeout = setTimeout(() => {
|
|
51
51
|
ws.terminate();
|
|
52
|
-
reject(
|
|
52
|
+
reject(
|
|
53
|
+
new APITimeoutError({
|
|
54
|
+
message: "WebSocket connection timeout",
|
|
55
|
+
options: { retryable: false }
|
|
56
|
+
})
|
|
57
|
+
);
|
|
53
58
|
}, options.timeout);
|
|
54
59
|
ws.once("open", () => {
|
|
55
60
|
clearTimeout(timeout);
|
|
56
61
|
resolve();
|
|
57
62
|
});
|
|
63
|
+
ws.once("unexpected-response", (_req, res) => {
|
|
64
|
+
clearTimeout(timeout);
|
|
65
|
+
ws.terminate();
|
|
66
|
+
const statusCode = res.statusCode ?? -1;
|
|
67
|
+
reject(
|
|
68
|
+
new APIStatusError({
|
|
69
|
+
message: `WebSocket connection rejected with status ${statusCode}`,
|
|
70
|
+
options: { statusCode, retryable: false }
|
|
71
|
+
})
|
|
72
|
+
);
|
|
73
|
+
});
|
|
58
74
|
ws.once("error", (err) => {
|
|
59
75
|
clearTimeout(timeout);
|
|
60
76
|
ws.terminate();
|
|
61
|
-
reject(err);
|
|
77
|
+
reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
|
|
62
78
|
});
|
|
63
79
|
});
|
|
64
80
|
return ws;
|
|
@@ -77,7 +93,9 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
77
93
|
}
|
|
78
94
|
});
|
|
79
95
|
socket.on("error", (err) => {
|
|
80
|
-
|
|
96
|
+
outputController == null ? void 0 : outputController.error(
|
|
97
|
+
new APIConnectionError({ message: `WebSocket error: ${err.message}` })
|
|
98
|
+
);
|
|
81
99
|
});
|
|
82
100
|
socket.on("close", (code, reason) => {
|
|
83
101
|
logger.debug({ code, reason: reason.toString() }, "WebSocket closed");
|
|
@@ -85,37 +103,19 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
85
103
|
}
|
|
86
104
|
async function ensureConnection() {
|
|
87
105
|
if (ws && ws.readyState === WebSocket.OPEN) return;
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
num_channels: 1,
|
|
99
|
-
threshold: options.threshold,
|
|
100
|
-
min_frames: options.minFrames,
|
|
101
|
-
encoding: "s16le"
|
|
102
|
-
}
|
|
103
|
-
});
|
|
104
|
-
ws.send(sessionCreateMsg);
|
|
105
|
-
return;
|
|
106
|
-
} catch (err) {
|
|
107
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
108
|
-
if (attempt < maxRetries) {
|
|
109
|
-
const delay = intervalForRetry(attempt);
|
|
110
|
-
logger.debug(
|
|
111
|
-
{ attempt, delay, err: lastError.message },
|
|
112
|
-
"WebSocket connection failed, retrying"
|
|
113
|
-
);
|
|
114
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
115
|
-
}
|
|
106
|
+
ws = await connectWebSocket(options);
|
|
107
|
+
setupMessageHandler(ws);
|
|
108
|
+
const sessionCreateMsg = JSON.stringify({
|
|
109
|
+
type: MSG_SESSION_CREATE,
|
|
110
|
+
settings: {
|
|
111
|
+
sample_rate: options.sampleRate,
|
|
112
|
+
num_channels: 1,
|
|
113
|
+
threshold: options.threshold,
|
|
114
|
+
min_frames: options.minFrames,
|
|
115
|
+
encoding: "s16le"
|
|
116
116
|
}
|
|
117
|
-
}
|
|
118
|
-
|
|
117
|
+
});
|
|
118
|
+
ws.send(sessionCreateMsg);
|
|
119
119
|
}
|
|
120
120
|
function handleMessage(message) {
|
|
121
121
|
const state = getState();
|
|
@@ -155,8 +155,8 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
155
155
|
"interruption detected"
|
|
156
156
|
);
|
|
157
157
|
const event = {
|
|
158
|
-
type: "
|
|
159
|
-
|
|
158
|
+
type: "overlapping_speech",
|
|
159
|
+
detectedAt: Date.now(),
|
|
160
160
|
isInterruption: true,
|
|
161
161
|
totalDurationInS: entry.totalDurationInS,
|
|
162
162
|
predictionDurationInS: entry.predictionDurationInS,
|
|
@@ -206,16 +206,17 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
206
206
|
break;
|
|
207
207
|
case MSG_ERROR:
|
|
208
208
|
outputController == null ? void 0 : outputController.error(
|
|
209
|
-
new
|
|
210
|
-
`LiveKit Adaptive Interruption error
|
|
211
|
-
|
|
209
|
+
new APIStatusError({
|
|
210
|
+
message: `LiveKit Adaptive Interruption error: ${message.message}`,
|
|
211
|
+
options: { statusCode: message.code ?? -1 }
|
|
212
|
+
})
|
|
212
213
|
);
|
|
213
214
|
break;
|
|
214
215
|
}
|
|
215
216
|
}
|
|
216
217
|
function sendAudioData(audioSlice) {
|
|
217
218
|
if (!ws || ws.readyState !== WebSocket.OPEN) {
|
|
218
|
-
throw new
|
|
219
|
+
throw new APIConnectionError({ message: "WebSocket not connected" });
|
|
219
220
|
}
|
|
220
221
|
const state = getState();
|
|
221
222
|
const createdAt = Math.floor(performance.now());
|
|
@@ -239,12 +240,8 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
239
240
|
const combined = new Uint8Array(8 + audioBytes.length);
|
|
240
241
|
combined.set(new Uint8Array(header), 0);
|
|
241
242
|
combined.set(audioBytes, 8);
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
onRequestSent == null ? void 0 : onRequestSent();
|
|
245
|
-
} catch (e) {
|
|
246
|
-
logger.error(e, `failed to send audio via websocket`);
|
|
247
|
-
}
|
|
243
|
+
ws.send(combined);
|
|
244
|
+
onRequestSent == null ? void 0 : onRequestSent();
|
|
248
245
|
}
|
|
249
246
|
function close() {
|
|
250
247
|
if ((ws == null ? void 0 : ws.readyState) === WebSocket.OPEN) {
|
|
@@ -274,10 +271,26 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
|
|
|
274
271
|
}
|
|
275
272
|
const state = getState();
|
|
276
273
|
if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;
|
|
274
|
+
if (options.timeout > 0) {
|
|
275
|
+
const now = performance.now();
|
|
276
|
+
for (const [, entry] of state.cache.entries()) {
|
|
277
|
+
if (entry.totalDurationInS !== 0) continue;
|
|
278
|
+
if (now - entry.createdAt > options.timeout) {
|
|
279
|
+
controller.error(
|
|
280
|
+
new APIStatusError({
|
|
281
|
+
message: `interruption inference timed out after ${((now - entry.createdAt) / 1e3).toFixed(1)}s (ws)`,
|
|
282
|
+
options: { statusCode: 408, retryable: false }
|
|
283
|
+
})
|
|
284
|
+
);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
break;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
277
290
|
try {
|
|
278
291
|
sendAudioData(chunk);
|
|
279
292
|
} catch (err) {
|
|
280
|
-
|
|
293
|
+
controller.error(err);
|
|
281
294
|
}
|
|
282
295
|
},
|
|
283
296
|
flush() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { intervalForRetry } from './defaults.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new Promise<void>((resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(new Error('WebSocket connection timeout'));\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(err);\n });\n });\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n logger.error({ err }, 'WebSocket error');\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<void> {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n const maxRetries = options.maxRetries ?? 3;\n let lastError: Error | null = null;\n\n for (let attempt = 0; attempt <= maxRetries; attempt++) {\n try {\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n // Send session.create message\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n return;\n } catch (err) {\n lastError = err instanceof Error ? err : new Error(String(err));\n if (attempt < maxRetries) {\n const delay = intervalForRetry(attempt);\n logger.debug(\n { attempt, delay, err: lastError.message },\n 'WebSocket connection failed, retrying',\n );\n await new Promise((resolve) => setTimeout(resolve, delay));\n }\n }\n }\n\n throw lastError ?? new Error('Failed to connect to WebSocket after retries');\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'user_overlapping_speech',\n timestamp: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new Error(\n `LiveKit Adaptive Interruption error${\n message.code !== undefined ? ` (${message.code})` : ''\n }: ${message.message}`,\n ),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new Error('WebSocket not connected');\n }\n\n const state = getState();\n // Use truncated timestamp consistently for both cache key and header\n // This ensures the server's response created_at matches our cache key\n const createdAt = Math.floor(performance.now());\n\n // Store the audio data in cache with truncated timestamp\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n // Create header: 8-byte little-endian uint64 timestamp (milliseconds as integer)\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n // Combine header and audio data\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n try {\n ws.send(combined);\n onRequestSent?.();\n } catch (e: unknown) {\n logger.error(e, `failed to send audio via websocket`);\n }\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection();\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n logger.error({ err }, 'Failed to send audio data over WebSocket');\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":"AAGA,SAAS,uBAAuB;AAChC,OAAO,eAAe;AACtB,SAAS,SAAS;AAClB,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAClC,SAAS,wBAAwB;AACjC,SAAS,8BAA8B;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,EAAE,mBAAmB,QAAQ;AAAA,EACnD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,EAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,EAAE,OAAO;AAAA,IAClB,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb,aAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,IAClD,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,GAAG;AAAA,IACZ,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,SAAS,IAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,aAAO,MAAM,EAAE,IAAI,GAAG,iBAAiB;AAAA,IACzC,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAU,KAAM;AAE5C,UAAM,aAAa,QAAQ,cAAc;AACzC,QAAI,YAA0B;AAE9B,aAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,UAAI;AACF,aAAK,MAAM,iBAAiB,OAAO;AACnC,4BAAoB,EAAE;AAGtB,cAAM,mBAAmB,KAAK,UAAU;AAAA,UACtC,MAAM;AAAA,UACN,UAAU;AAAA,YACR,aAAa,QAAQ;AAAA,YACrB,cAAc;AAAA,YACd,WAAW,QAAQ;AAAA,YACnB,YAAY,QAAQ;AAAA,YACpB,UAAU;AAAA,UACZ;AAAA,QACF,CAAC;AACD,WAAG,KAAK,gBAAgB;AACxB;AAAA,MACF,SAAS,KAAK;AACZ,oBAAY,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC9D,YAAI,UAAU,YAAY;AACxB,gBAAM,QAAQ,iBAAiB,OAAO;AACtC,iBAAO;AAAA,YACL,EAAE,SAAS,OAAO,KAAK,UAAU,QAAQ;AAAA,YACzC;AAAA,UACF;AACA,gBAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,IACF;AAEA,UAAM,aAAa,IAAI,MAAM,8CAA8C;AAAA,EAC7E;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,WAAW,KAAK,IAAI;AAAA,YACpB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI;AAAA,YACF,sCACE,QAAQ,SAAS,SAAY,KAAK,QAAQ,IAAI,MAAM,EACtD,KAAK,QAAQ,OAAO;AAAA,UACtB;AAAA;AAEF;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAU,MAAM;AAC3C,YAAM,IAAI,MAAM,yBAAyB;AAAA,IAC3C;AAEA,UAAM,QAAQ,SAAS;AAGvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAG9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uBAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAGA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAGjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,QAAI;AACF,SAAG,KAAK,QAAQ;AAChB;AAAA,IACF,SAAS,GAAY;AACnB,aAAO,MAAM,GAAG,oCAAoC;AAAA,IACtD;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,iBAAO,MAAM,EAAE,IAAI,GAAG,0CAA0C;AAAA,QAClE;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new Promise<void>((resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(\n new APITimeoutError({\n message: 'WebSocket connection timeout',\n options: { retryable: false },\n }),\n );\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('unexpected-response', (_req, res) => {\n clearTimeout(timeout);\n ws.terminate();\n const statusCode = res.statusCode ?? -1;\n reject(\n new APIStatusError({\n message: `WebSocket connection rejected with status ${statusCode}`,\n options: { statusCode, retryable: false },\n }),\n );\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));\n });\n });\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n outputController?.error(\n new APIConnectionError({ message: `WebSocket error: ${err.message}` }),\n );\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<void> {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new APIStatusError({\n message: `LiveKit Adaptive Interruption error: ${message.message}`,\n options: { statusCode: message.code ?? -1 },\n }),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new APIConnectionError({ message: 'WebSocket not connected' });\n }\n\n const state = getState();\n const createdAt = Math.floor(performance.now());\n\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n ws.send(combined);\n onRequestSent?.();\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection();\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n if (options.timeout > 0) {\n const now = performance.now();\n for (const [, entry] of state.cache.entries()) {\n if (entry.totalDurationInS !== 0) continue;\n if (now - entry.createdAt > options.timeout) {\n controller.error(\n new APIStatusError({\n message: `interruption inference timed out after ${((now - entry.createdAt) / 1000).toFixed(1)}s (ws)`,\n options: { statusCode: 408, retryable: false },\n }),\n );\n return;\n }\n break;\n }\n }\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n controller.error(err);\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":"AAGA,SAAS,uBAAuB;AAChC,OAAO,eAAe;AACtB,SAAS,SAAS;AAClB,SAAS,oBAAoB,gBAAgB,uBAAuB;AACpE,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAClC,SAAS,8BAA8B;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,EAAE,mBAAmB,QAAQ;AAAA,EACnD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,EAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,EAAE,OAAO;AAAA,IAClB,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb;AAAA,QACE,IAAI,gBAAgB;AAAA,UAClB,SAAS;AAAA,UACT,SAAS,EAAE,WAAW,MAAM;AAAA,QAC9B,CAAC;AAAA,MACH;AAAA,IACF,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,uBAAuB,CAAC,MAAM,QAAQ;AAC5C,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,YAAM,aAAa,IAAI,cAAc;AACrC;AAAA,QACE,IAAI,eAAe;AAAA,UACjB,SAAS,6CAA6C,UAAU;AAAA,UAChE,SAAS,EAAE,YAAY,WAAW,MAAM;AAAA,QAC1C,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,IAAI,mBAAmB,EAAE,SAAS,+BAA+B,IAAI,OAAO,GAAG,CAAC,CAAC;AAAA,IAC1F,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,SAAS,IAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,2DAAkB;AAAA,QAChB,IAAI,mBAAmB,EAAE,SAAS,oBAAoB,IAAI,OAAO,GAAG,CAAC;AAAA;AAAA,IAEzE,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAU,KAAM;AAE5C,SAAK,MAAM,iBAAiB,OAAO;AACnC,wBAAoB,EAAE;AAEtB,UAAM,mBAAmB,KAAK,UAAU;AAAA,MACtC,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa,QAAQ;AAAA,QACrB,cAAc;AAAA,QACd,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB,UAAU;AAAA,MACZ;AAAA,IACF,CAAC;AACD,OAAG,KAAK,gBAAgB;AAAA,EAC1B;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,YAAY,KAAK,IAAI;AAAA,YACrB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI,eAAe;AAAA,YACjB,SAAS,wCAAwC,QAAQ,OAAO;AAAA,YAChE,SAAS,EAAE,YAAY,QAAQ,QAAQ,GAAG;AAAA,UAC5C,CAAC;AAAA;AAEH;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAU,MAAM;AAC3C,YAAM,IAAI,mBAAmB,EAAE,SAAS,0BAA0B,CAAC;AAAA,IACrE;AAEA,UAAM,QAAQ,SAAS;AACvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAE9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uBAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAEjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,OAAG,KAAK,QAAQ;AAChB;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI,QAAQ,UAAU,GAAG;AACvB,gBAAM,MAAM,YAAY,IAAI;AAC5B,qBAAW,CAAC,EAAE,KAAK,KAAK,MAAM,MAAM,QAAQ,GAAG;AAC7C,gBAAI,MAAM,qBAAqB,EAAG;AAClC,gBAAI,MAAM,MAAM,YAAY,QAAQ,SAAS;AAC3C,yBAAW;AAAA,gBACT,IAAI,eAAe;AAAA,kBACjB,SAAS,4CAA4C,MAAM,MAAM,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,kBAC9F,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,gBAC/C,CAAC;AAAA,cACH;AACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF;AAEA,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":[]}
|