@livekit/agents 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/dist/_exceptions.cjs.map +1 -1
  2. package/dist/_exceptions.d.ts.map +1 -1
  3. package/dist/_exceptions.js.map +1 -1
  4. package/dist/audio.cjs +10 -0
  5. package/dist/audio.cjs.map +1 -1
  6. package/dist/audio.d.cts +1 -1
  7. package/dist/audio.d.ts +1 -1
  8. package/dist/audio.d.ts.map +1 -1
  9. package/dist/audio.js +10 -0
  10. package/dist/audio.js.map +1 -1
  11. package/dist/beta/workflows/task_group.cjs +7 -4
  12. package/dist/beta/workflows/task_group.cjs.map +1 -1
  13. package/dist/beta/workflows/task_group.d.ts.map +1 -1
  14. package/dist/beta/workflows/task_group.js +7 -4
  15. package/dist/beta/workflows/task_group.js.map +1 -1
  16. package/dist/inference/api_protos.d.cts +26 -26
  17. package/dist/inference/api_protos.d.ts +26 -26
  18. package/dist/inference/interruption/http_transport.cjs.map +1 -1
  19. package/dist/inference/interruption/http_transport.d.cts +3 -1
  20. package/dist/inference/interruption/http_transport.d.ts +3 -1
  21. package/dist/inference/interruption/http_transport.d.ts.map +1 -1
  22. package/dist/inference/interruption/http_transport.js.map +1 -1
  23. package/dist/inference/interruption/ws_transport.cjs +37 -32
  24. package/dist/inference/interruption/ws_transport.cjs.map +1 -1
  25. package/dist/inference/interruption/ws_transport.d.ts.map +1 -1
  26. package/dist/inference/interruption/ws_transport.js +37 -32
  27. package/dist/inference/interruption/ws_transport.js.map +1 -1
  28. package/dist/inference/tts.cjs +14 -1
  29. package/dist/inference/tts.cjs.map +1 -1
  30. package/dist/inference/tts.d.cts +42 -4
  31. package/dist/inference/tts.d.ts +42 -4
  32. package/dist/inference/tts.d.ts.map +1 -1
  33. package/dist/inference/tts.js +24 -3
  34. package/dist/inference/tts.js.map +1 -1
  35. package/dist/inference/tts.test.cjs +72 -0
  36. package/dist/inference/tts.test.cjs.map +1 -1
  37. package/dist/inference/tts.test.js +72 -0
  38. package/dist/inference/tts.test.js.map +1 -1
  39. package/dist/ipc/job_proc_lazy_main.cjs +7 -2
  40. package/dist/ipc/job_proc_lazy_main.cjs.map +1 -1
  41. package/dist/ipc/job_proc_lazy_main.js +7 -2
  42. package/dist/ipc/job_proc_lazy_main.js.map +1 -1
  43. package/dist/ipc/supervised_proc.cjs +4 -1
  44. package/dist/ipc/supervised_proc.cjs.map +1 -1
  45. package/dist/ipc/supervised_proc.d.ts.map +1 -1
  46. package/dist/ipc/supervised_proc.js +4 -1
  47. package/dist/ipc/supervised_proc.js.map +1 -1
  48. package/dist/ipc/supervised_proc.test.cjs +82 -0
  49. package/dist/ipc/supervised_proc.test.cjs.map +1 -1
  50. package/dist/ipc/supervised_proc.test.js +82 -0
  51. package/dist/ipc/supervised_proc.test.js.map +1 -1
  52. package/dist/job.cjs +2 -1
  53. package/dist/job.cjs.map +1 -1
  54. package/dist/job.d.ts.map +1 -1
  55. package/dist/job.js +2 -1
  56. package/dist/job.js.map +1 -1
  57. package/dist/llm/chat_context.cjs +102 -31
  58. package/dist/llm/chat_context.cjs.map +1 -1
  59. package/dist/llm/chat_context.d.ts.map +1 -1
  60. package/dist/llm/chat_context.js +102 -31
  61. package/dist/llm/chat_context.js.map +1 -1
  62. package/dist/llm/chat_context.test.cjs +123 -5
  63. package/dist/llm/chat_context.test.cjs.map +1 -1
  64. package/dist/llm/chat_context.test.js +123 -5
  65. package/dist/llm/chat_context.test.js.map +1 -1
  66. package/dist/llm/fallback_adapter.cjs +2 -0
  67. package/dist/llm/fallback_adapter.cjs.map +1 -1
  68. package/dist/llm/fallback_adapter.d.ts.map +1 -1
  69. package/dist/llm/fallback_adapter.js +2 -0
  70. package/dist/llm/fallback_adapter.js.map +1 -1
  71. package/dist/llm/index.cjs +2 -0
  72. package/dist/llm/index.cjs.map +1 -1
  73. package/dist/llm/index.d.cts +1 -1
  74. package/dist/llm/index.d.ts +1 -1
  75. package/dist/llm/index.d.ts.map +1 -1
  76. package/dist/llm/index.js +2 -0
  77. package/dist/llm/index.js.map +1 -1
  78. package/dist/llm/utils.cjs +89 -0
  79. package/dist/llm/utils.cjs.map +1 -1
  80. package/dist/llm/utils.d.cts +8 -0
  81. package/dist/llm/utils.d.ts +8 -0
  82. package/dist/llm/utils.d.ts.map +1 -1
  83. package/dist/llm/utils.js +88 -0
  84. package/dist/llm/utils.js.map +1 -1
  85. package/dist/llm/utils.test.cjs +90 -0
  86. package/dist/llm/utils.test.cjs.map +1 -1
  87. package/dist/llm/utils.test.js +98 -2
  88. package/dist/llm/utils.test.js.map +1 -1
  89. package/dist/stt/stt.cjs +8 -0
  90. package/dist/stt/stt.cjs.map +1 -1
  91. package/dist/stt/stt.d.cts +8 -0
  92. package/dist/stt/stt.d.ts +8 -0
  93. package/dist/stt/stt.d.ts.map +1 -1
  94. package/dist/stt/stt.js +8 -0
  95. package/dist/stt/stt.js.map +1 -1
  96. package/dist/tts/fallback_adapter.cjs +6 -0
  97. package/dist/tts/fallback_adapter.cjs.map +1 -1
  98. package/dist/tts/fallback_adapter.d.ts.map +1 -1
  99. package/dist/tts/fallback_adapter.js +6 -0
  100. package/dist/tts/fallback_adapter.js.map +1 -1
  101. package/dist/typed_promise.cjs +48 -0
  102. package/dist/typed_promise.cjs.map +1 -0
  103. package/dist/typed_promise.d.cts +24 -0
  104. package/dist/typed_promise.d.ts +24 -0
  105. package/dist/typed_promise.d.ts.map +1 -0
  106. package/dist/typed_promise.js +28 -0
  107. package/dist/typed_promise.js.map +1 -0
  108. package/dist/utils.cjs +30 -2
  109. package/dist/utils.cjs.map +1 -1
  110. package/dist/utils.d.cts +18 -0
  111. package/dist/utils.d.ts +18 -0
  112. package/dist/utils.d.ts.map +1 -1
  113. package/dist/utils.js +27 -2
  114. package/dist/utils.js.map +1 -1
  115. package/dist/version.cjs +1 -1
  116. package/dist/version.js +1 -1
  117. package/dist/voice/agent_activity.cjs +10 -0
  118. package/dist/voice/agent_activity.cjs.map +1 -1
  119. package/dist/voice/agent_activity.d.ts.map +1 -1
  120. package/dist/voice/agent_activity.js +11 -0
  121. package/dist/voice/agent_activity.js.map +1 -1
  122. package/dist/voice/agent_session.cjs +1 -1
  123. package/dist/voice/agent_session.cjs.map +1 -1
  124. package/dist/voice/agent_session.d.cts +4 -2
  125. package/dist/voice/agent_session.d.ts +4 -2
  126. package/dist/voice/agent_session.d.ts.map +1 -1
  127. package/dist/voice/agent_session.js +1 -1
  128. package/dist/voice/agent_session.js.map +1 -1
  129. package/dist/voice/events.cjs +11 -0
  130. package/dist/voice/events.cjs.map +1 -1
  131. package/dist/voice/events.d.cts +12 -1
  132. package/dist/voice/events.d.ts +12 -1
  133. package/dist/voice/events.d.ts.map +1 -1
  134. package/dist/voice/events.js +10 -0
  135. package/dist/voice/events.js.map +1 -1
  136. package/dist/voice/generation.cjs +23 -4
  137. package/dist/voice/generation.cjs.map +1 -1
  138. package/dist/voice/generation.d.ts.map +1 -1
  139. package/dist/voice/generation.js +32 -5
  140. package/dist/voice/generation.js.map +1 -1
  141. package/dist/voice/generation_tts_timeout.test.cjs +85 -0
  142. package/dist/voice/generation_tts_timeout.test.cjs.map +1 -0
  143. package/dist/voice/generation_tts_timeout.test.js +84 -0
  144. package/dist/voice/generation_tts_timeout.test.js.map +1 -0
  145. package/dist/voice/index.cjs.map +1 -1
  146. package/dist/voice/index.d.cts +1 -1
  147. package/dist/voice/index.d.ts +1 -1
  148. package/dist/voice/index.d.ts.map +1 -1
  149. package/dist/voice/index.js +3 -1
  150. package/dist/voice/index.js.map +1 -1
  151. package/dist/voice/recorder_io/recorder_io.cjs +1 -2
  152. package/dist/voice/recorder_io/recorder_io.cjs.map +1 -1
  153. package/dist/voice/recorder_io/recorder_io.d.ts.map +1 -1
  154. package/dist/voice/recorder_io/recorder_io.js +2 -3
  155. package/dist/voice/recorder_io/recorder_io.js.map +1 -1
  156. package/dist/voice/report.cjs +1 -1
  157. package/dist/voice/report.cjs.map +1 -1
  158. package/dist/voice/report.js +1 -1
  159. package/dist/voice/report.js.map +1 -1
  160. package/dist/voice/report.test.cjs +70 -0
  161. package/dist/voice/report.test.cjs.map +1 -1
  162. package/dist/voice/report.test.js +70 -0
  163. package/dist/voice/report.test.js.map +1 -1
  164. package/dist/voice/room_io/room_io.cjs +5 -1
  165. package/dist/voice/room_io/room_io.cjs.map +1 -1
  166. package/dist/voice/room_io/room_io.d.ts.map +1 -1
  167. package/dist/voice/room_io/room_io.js +5 -1
  168. package/dist/voice/room_io/room_io.js.map +1 -1
  169. package/dist/voice/room_io/room_io.test.cjs +18 -0
  170. package/dist/voice/room_io/room_io.test.cjs.map +1 -0
  171. package/dist/voice/room_io/room_io.test.js +17 -0
  172. package/dist/voice/room_io/room_io.test.js.map +1 -0
  173. package/package.json +4 -2
  174. package/src/_exceptions.ts +5 -0
  175. package/src/audio.ts +12 -1
  176. package/src/beta/workflows/task_group.ts +14 -5
  177. package/src/inference/interruption/http_transport.ts +2 -1
  178. package/src/inference/interruption/ws_transport.ts +44 -34
  179. package/src/inference/tts.test.ts +87 -0
  180. package/src/inference/tts.ts +71 -9
  181. package/src/ipc/job_proc_lazy_main.ts +7 -2
  182. package/src/ipc/supervised_proc.test.ts +96 -0
  183. package/src/ipc/supervised_proc.ts +8 -1
  184. package/src/job.ts +1 -0
  185. package/src/llm/chat_context.test.ts +137 -5
  186. package/src/llm/chat_context.ts +119 -38
  187. package/src/llm/fallback_adapter.ts +5 -2
  188. package/src/llm/index.ts +2 -0
  189. package/src/llm/utils.test.ts +103 -2
  190. package/src/llm/utils.ts +128 -0
  191. package/src/stt/stt.ts +9 -1
  192. package/src/tts/fallback_adapter.ts +9 -2
  193. package/src/typed_promise.ts +67 -0
  194. package/src/utils.ts +45 -2
  195. package/src/voice/agent_activity.ts +11 -0
  196. package/src/voice/agent_session.ts +13 -7
  197. package/src/voice/events.ts +21 -0
  198. package/src/voice/generation.ts +35 -8
  199. package/src/voice/generation_tts_timeout.test.ts +112 -0
  200. package/src/voice/index.ts +6 -1
  201. package/src/voice/recorder_io/recorder_io.ts +2 -7
  202. package/src/voice/report.test.ts +78 -0
  203. package/src/voice/report.ts +1 -1
  204. package/src/voice/room_io/room_io.test.ts +38 -0
  205. package/src/voice/room_io/room_io.ts +7 -2
@@ -36,6 +36,7 @@ var import_ws = __toESM(require("ws"), 1);
36
36
  var import_zod = require("zod");
37
37
  var import_exceptions = require("../../_exceptions.cjs");
38
38
  var import_log = require("../../log.cjs");
39
+ var import_typed_promise = __toESM(require("../../typed_promise.cjs"), 1);
39
40
  var import_utils = require("../utils.cjs");
40
41
  var import_interruption_cache_entry = require("./interruption_cache_entry.cjs");
41
42
  const MSG_SESSION_CREATE = "session.create";
@@ -79,37 +80,39 @@ async function connectWebSocket(options) {
79
80
  const ws = new import_ws.default(url, {
80
81
  headers: { Authorization: `Bearer ${token}` }
81
82
  });
82
- await new Promise((resolve, reject) => {
83
- const timeout = setTimeout(() => {
84
- ws.terminate();
85
- reject(
86
- new import_exceptions.APITimeoutError({
87
- message: "WebSocket connection timeout",
88
- options: { retryable: false }
89
- })
90
- );
91
- }, options.timeout);
92
- ws.once("open", () => {
93
- clearTimeout(timeout);
94
- resolve();
95
- });
96
- ws.once("unexpected-response", (_req, res) => {
97
- clearTimeout(timeout);
98
- ws.terminate();
99
- const statusCode = res.statusCode ?? -1;
100
- reject(
101
- new import_exceptions.APIStatusError({
102
- message: `WebSocket connection rejected with status ${statusCode}`,
103
- options: { statusCode, retryable: false }
104
- })
105
- );
106
- });
107
- ws.once("error", (err) => {
108
- clearTimeout(timeout);
109
- ws.terminate();
110
- reject(new import_exceptions.APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
111
- });
112
- });
83
+ await new import_typed_promise.default(
84
+ (resolve, reject) => {
85
+ const timeout = setTimeout(() => {
86
+ ws.terminate();
87
+ reject(
88
+ new import_exceptions.APITimeoutError({
89
+ message: "WebSocket connection timeout",
90
+ options: { retryable: false }
91
+ })
92
+ );
93
+ }, options.timeout);
94
+ ws.once("open", () => {
95
+ clearTimeout(timeout);
96
+ resolve();
97
+ });
98
+ ws.once("unexpected-response", (_req, res) => {
99
+ clearTimeout(timeout);
100
+ ws.terminate();
101
+ const statusCode = res.statusCode ?? -1;
102
+ reject(
103
+ new import_exceptions.APIStatusError({
104
+ message: `WebSocket connection rejected with status ${statusCode}`,
105
+ options: { statusCode, retryable: false }
106
+ })
107
+ );
108
+ });
109
+ ws.once("error", (err) => {
110
+ clearTimeout(timeout);
111
+ ws.terminate();
112
+ reject(new import_exceptions.APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
113
+ });
114
+ }
115
+ );
113
116
  return ws;
114
117
  }
115
118
  function createWsTransport(options, getState, setState, updateUserSpeakingSpan, onRequestSent, getAndResetNumRequests) {
@@ -295,7 +298,9 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
295
298
  {
296
299
  async start(controller) {
297
300
  outputController = controller;
298
- await ensureConnection();
301
+ await ensureConnection().catch((e) => {
302
+ controller.error(e);
303
+ });
299
304
  },
300
305
  transform(chunk, controller) {
301
306
  if (!(chunk instanceof Int16Array)) {
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new Promise<void>((resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(\n new APITimeoutError({\n message: 'WebSocket connection timeout',\n options: { retryable: false },\n }),\n );\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('unexpected-response', (_req, res) => {\n clearTimeout(timeout);\n ws.terminate();\n const statusCode = res.statusCode ?? -1;\n reject(\n new APIStatusError({\n message: `WebSocket connection rejected with status ${statusCode}`,\n options: { statusCode, retryable: false },\n }),\n );\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));\n });\n });\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n outputController?.error(\n new APIConnectionError({ message: `WebSocket error: ${err.message}` }),\n );\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<void> {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new APIStatusError({\n message: `LiveKit Adaptive Interruption error: ${message.message}`,\n options: { statusCode: message.code ?? -1 },\n }),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new APIConnectionError({ message: 'WebSocket not connected' });\n }\n\n const state = getState();\n const createdAt = Math.floor(performance.now());\n\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n ws.send(combined);\n onRequestSent?.();\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection();\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n if (options.timeout > 0) {\n const now = performance.now();\n for (const [, entry] of state.cache.entries()) {\n if (entry.totalDurationInS !== 0) continue;\n if (now - entry.createdAt > options.timeout) {\n controller.error(\n new APIStatusError({\n message: `interruption inference timed out after ${((now - entry.createdAt) / 1000).toFixed(1)}s (ws)`,\n options: { statusCode: 408, retryable: false },\n }),\n );\n return;\n }\n break;\n }\n }\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n controller.error(err);\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAAgC;AAChC,gBAAsB;AACtB,iBAAkB;AAClB,wBAAoE;AACpE,iBAAoB;AACpB,mBAAkC;AAClC,sCAAuC;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,aAAE,mBAAmB,QAAQ;AAAA,EACnD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,aAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,aAAE,OAAO;AAAA,IAClB,MAAM,aAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,aAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,UAAM,gCAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAAA,QAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb;AAAA,QACE,IAAI,kCAAgB;AAAA,UAClB,SAAS;AAAA,UACT,SAAS,EAAE,WAAW,MAAM;AAAA,QAC9B,CAAC;AAAA,MACH;AAAA,IACF,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,uBAAuB,CAAC,MAAM,QAAQ;AAC5C,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,YAAM,aAAa,IAAI,cAAc;AACrC;AAAA,QACE,IAAI,iCAAe;AAAA,UACjB,SAAS,6CAA6C,UAAU;AAAA,UAChE,SAAS,EAAE,YAAY,WAAW,MAAM;AAAA,QAC1C,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,IAAI,qCAAmB,EAAE,SAAS,+BAA+B,IAAI,OAAO,GAAG,CAAC,CAAC;AAAA,IAC1F,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,aAAS,gBAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,2DAAkB;AAAA,QAChB,IAAI,qCAAmB,EAAE,SAAS,oBAAoB,IAAI,OAAO,GAAG,CAAC;AAAA;AAAA,IAEzE,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAAA,QAAU,KAAM;AAE5C,SAAK,MAAM,iBAAiB,OAAO;AACnC,wBAAoB,EAAE;AAEtB,UAAM,mBAAmB,KAAK,UAAU;AAAA,MACtC,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa,QAAQ;AAAA,QACrB,cAAc;AAAA,QACd,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB,UAAU;AAAA,MACZ;AAAA,IACF,CAAC;AACD,OAAG,KAAK,gBAAgB;AAAA,EAC1B;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,YAAY,KAAK,IAAI;AAAA,YACrB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI,iCAAe;AAAA,YACjB,SAAS,wCAAwC,QAAQ,OAAO;AAAA,YAChE,SAAS,EAAE,YAAY,QAAQ,QAAQ,GAAG;AAAA,UAC5C,CAAC;AAAA;AAEH;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAAA,QAAU,MAAM;AAC3C,YAAM,IAAI,qCAAmB,EAAE,SAAS,0BAA0B,CAAC;AAAA,IACrE;AAEA,UAAM,QAAQ,SAAS;AACvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAE9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uDAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAEjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,OAAG,KAAK,QAAQ;AAChB;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAAA,QAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI,QAAQ,UAAU,GAAG;AACvB,gBAAM,MAAM,YAAY,IAAI;AAC5B,qBAAW,CAAC,EAAE,KAAK,KAAK,MAAM,MAAM,QAAQ,GAAG;AAC7C,gBAAI,MAAM,qBAAqB,EAAG;AAClC,gBAAI,MAAM,MAAM,YAAY,QAAQ,SAAS;AAC3C,yBAAW;AAAA,gBACT,IAAI,iCAAe;AAAA,kBACjB,SAAS,4CAA4C,MAAM,MAAM,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,kBAC9F,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,gBAC/C,CAAC;AAAA,cACH;AACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF;AAEA,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":["WebSocket"]}
1
+ {"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Throws } from '@livekit/throws-transformer/throws';\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport TypedPromise from '../../typed_promise.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(\n options: WsTransportOptions,\n): Promise<Throws<WebSocket, APIStatusError | APITimeoutError | APIConnectionError>> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new TypedPromise<void, APIStatusError | APITimeoutError | APIConnectionError>(\n (resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(\n new APITimeoutError({\n message: 'WebSocket connection timeout',\n options: { retryable: false },\n }),\n );\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('unexpected-response', (_req, res) => {\n clearTimeout(timeout);\n ws.terminate();\n const statusCode = res.statusCode ?? -1;\n reject(\n new APIStatusError({\n message: `WebSocket connection rejected with status ${statusCode}`,\n options: { statusCode, retryable: false },\n }),\n );\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));\n });\n },\n );\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n outputController?.error(\n new APIConnectionError({ message: `WebSocket error: ${err.message}` }),\n );\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<\n Throws<void, APIStatusError | APITimeoutError | APIConnectionError>\n > {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new APIStatusError({\n message: `LiveKit Adaptive Interruption error: ${message.message}`,\n options: { statusCode: message.code ?? -1 },\n }),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new APIConnectionError({ message: 'WebSocket not connected' });\n }\n\n const state = getState();\n const createdAt = Math.floor(performance.now());\n\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n ws.send(combined);\n onRequestSent?.();\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection().catch((e) => {\n controller.error(e);\n });\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n if (options.timeout > 0) {\n const now = performance.now();\n for (const [, entry] of state.cache.entries()) {\n if (entry.totalDurationInS !== 0) continue;\n if (now - entry.createdAt > options.timeout) {\n controller.error(\n new APIStatusError({\n message: `interruption inference timed out after ${((now - entry.createdAt) / 1000).toFixed(1)}s (ws)`,\n options: { statusCode: 408, retryable: false },\n }),\n );\n return;\n }\n break;\n }\n }\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n controller.error(err);\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,iBAAgC;AAChC,gBAAsB;AACtB,iBAAkB;AAClB,wBAAoE;AACpE,iBAAoB;AACpB,2BAAyB;AACzB,mBAAkC;AAClC,sCAAuC;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,aAAE,mBAAmB,QAAQ;AAAA,EACnD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,aAAE,OAAO;AAAA,IACrB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,aAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,aAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,aAAE,OAAO;AAAA,IACP,MAAM,aAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,aAAE,OAAO;AAAA,IAClB,MAAM,aAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,aAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBACb,SACmF;AACnF,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,UAAM,gCAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAAA,QAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,qBAAAC;AAAA,IACR,CAAC,SAAS,WAAW;AACnB,YAAM,UAAU,WAAW,MAAM;AAC/B,WAAG,UAAU;AACb;AAAA,UACE,IAAI,kCAAgB;AAAA,YAClB,SAAS;AAAA,YACT,SAAS,EAAE,WAAW,MAAM;AAAA,UAC9B,CAAC;AAAA,QACH;AAAA,MACF,GAAG,QAAQ,OAAO;AAClB,SAAG,KAAK,QAAQ,MAAM;AACpB,qBAAa,OAAO;AACpB,gBAAQ;AAAA,MACV,CAAC;AACD,SAAG,KAAK,uBAAuB,CAAC,MAAM,QAAQ;AAC5C,qBAAa,OAAO;AACpB,WAAG,UAAU;AACb,cAAM,aAAa,IAAI,cAAc;AACrC;AAAA,UACE,IAAI,iCAAe;AAAA,YACjB,SAAS,6CAA6C,UAAU;AAAA,YAChE,SAAS,EAAE,YAAY,WAAW,MAAM;AAAA,UAC1C,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AACD,SAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,qBAAa,OAAO;AACpB,WAAG,UAAU;AACb,eAAO,IAAI,qCAAmB,EAAE,SAAS,+BAA+B,IAAI,OAAO,GAAG,CAAC,CAAC;AAAA,MAC1F,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,aAAS,gBAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,2DAAkB;AAAA,QAChB,IAAI,qCAAmB,EAAE,SAAS,oBAAoB,IAAI,OAAO,GAAG,CAAC;AAAA;AAAA,IAEzE,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAEb;AACA,QAAI,MAAM,GAAG,eAAe,UAAAD,QAAU,KAAM;AAE5C,SAAK,MAAM,iBAAiB,OAAO;AACnC,wBAAoB,EAAE;AAEtB,UAAM,mBAAmB,KAAK,UAAU;AAAA,MACtC,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa,QAAQ;AAAA,QACrB,cAAc;AAAA,QACd,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB,UAAU;AAAA,MACZ;AAAA,IACF,CAAC;AACD,OAAG,KAAK,gBAAgB;AAAA,EAC1B;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,YAAY,KAAK,IAAI;AAAA,YACrB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI,iCAAe;AAAA,YACjB,SAAS,wCAAwC,QAAQ,OAAO;AAAA,YAChE,SAAS,EAAE,YAAY,QAAQ,QAAQ,GAAG;AAAA,UAC5C,CAAC;AAAA;AAEH;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAAA,QAAU,MAAM;AAC3C,YAAM,IAAI,qCAAmB,EAAE,SAAS,0BAA0B,CAAC;AAAA,IACrE;AAEA,UAAM,QAAQ,SAAS;AACvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAE9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uDAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAEjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,OAAG,KAAK,QAAQ;AAChB;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAAA,QAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM;AACpC,qBAAW,MAAM,CAAC;AAAA,QACpB,CAAC;AAAA,MACH;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI,QAAQ,UAAU,GAAG;AACvB,gBAAM,MAAM,YAAY,IAAI;AAC5B,qBAAW,CAAC,EAAE,KAAK,KAAK,MAAM,MAAM,QAAQ,GAAG;AAC7C,gBAAI,MAAM,qBAAqB,EAAG;AAClC,gBAAI,MAAM,MAAM,YAAY,QAAQ,SAAS;AAC3C,yBAAW;AAAA,gBACT,IAAI,iCAAe;AAAA,kBACjB,SAAS,4CAA4C,MAAM,MAAM,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,kBAC9F,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,gBAC/C,CAAC;AAAA,cACH;AACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF;AAEA,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":["WebSocket","TypedPromise"]}
@@ -1 +1 @@
1
- {"version":3,"file":"ws_transport.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/ws_transport.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAM7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAW/C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CACrD;AA+ED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,eAAe,CAAC,UAAU,GAAG,sBAAsB,EAAE,sBAAsB,CAAC,CAAC;IACxF,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,MAAM,gBAAgB,EAChC,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC,KAAK,IAAI,EACtD,sBAAsB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EAChE,aAAa,CAAC,EAAE,MAAM,IAAI,EAC1B,sBAAsB,CAAC,EAAE,MAAM,MAAM,GACpC,iBAAiB,CA8QnB"}
1
+ {"version":3,"file":"ws_transport.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/ws_transport.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAO7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAW/C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CACrD;AAmFD,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,eAAe,CAAC,UAAU,GAAG,sBAAsB,EAAE,sBAAsB,CAAC,CAAC;IACxF,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,kBAAkB,EAC3B,QAAQ,EAAE,MAAM,gBAAgB,EAChC,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC,KAAK,IAAI,EACtD,sBAAsB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EAChE,aAAa,CAAC,EAAE,MAAM,IAAI,EAC1B,sBAAsB,CAAC,EAAE,MAAM,MAAM,GACpC,iBAAiB,CAkRnB"}
@@ -3,6 +3,7 @@ import WebSocket from "ws";
3
3
  import { z } from "zod";
4
4
  import { APIConnectionError, APIStatusError, APITimeoutError } from "../../_exceptions.js";
5
5
  import { log } from "../../log.js";
6
+ import TypedPromise from "../../typed_promise.js";
6
7
  import { createAccessToken } from "../utils.js";
7
8
  import { InterruptionCacheEntry } from "./interruption_cache_entry.js";
8
9
  const MSG_SESSION_CREATE = "session.create";
@@ -46,37 +47,39 @@ async function connectWebSocket(options) {
46
47
  const ws = new WebSocket(url, {
47
48
  headers: { Authorization: `Bearer ${token}` }
48
49
  });
49
- await new Promise((resolve, reject) => {
50
- const timeout = setTimeout(() => {
51
- ws.terminate();
52
- reject(
53
- new APITimeoutError({
54
- message: "WebSocket connection timeout",
55
- options: { retryable: false }
56
- })
57
- );
58
- }, options.timeout);
59
- ws.once("open", () => {
60
- clearTimeout(timeout);
61
- resolve();
62
- });
63
- ws.once("unexpected-response", (_req, res) => {
64
- clearTimeout(timeout);
65
- ws.terminate();
66
- const statusCode = res.statusCode ?? -1;
67
- reject(
68
- new APIStatusError({
69
- message: `WebSocket connection rejected with status ${statusCode}`,
70
- options: { statusCode, retryable: false }
71
- })
72
- );
73
- });
74
- ws.once("error", (err) => {
75
- clearTimeout(timeout);
76
- ws.terminate();
77
- reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
78
- });
79
- });
50
+ await new TypedPromise(
51
+ (resolve, reject) => {
52
+ const timeout = setTimeout(() => {
53
+ ws.terminate();
54
+ reject(
55
+ new APITimeoutError({
56
+ message: "WebSocket connection timeout",
57
+ options: { retryable: false }
58
+ })
59
+ );
60
+ }, options.timeout);
61
+ ws.once("open", () => {
62
+ clearTimeout(timeout);
63
+ resolve();
64
+ });
65
+ ws.once("unexpected-response", (_req, res) => {
66
+ clearTimeout(timeout);
67
+ ws.terminate();
68
+ const statusCode = res.statusCode ?? -1;
69
+ reject(
70
+ new APIStatusError({
71
+ message: `WebSocket connection rejected with status ${statusCode}`,
72
+ options: { statusCode, retryable: false }
73
+ })
74
+ );
75
+ });
76
+ ws.once("error", (err) => {
77
+ clearTimeout(timeout);
78
+ ws.terminate();
79
+ reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
80
+ });
81
+ }
82
+ );
80
83
  return ws;
81
84
  }
82
85
  function createWsTransport(options, getState, setState, updateUserSpeakingSpan, onRequestSent, getAndResetNumRequests) {
@@ -262,7 +265,9 @@ function createWsTransport(options, getState, setState, updateUserSpeakingSpan,
262
265
  {
263
266
  async start(controller) {
264
267
  outputController = controller;
265
- await ensureConnection();
268
+ await ensureConnection().catch((e) => {
269
+ controller.error(e);
270
+ });
266
271
  },
267
272
  transform(chunk, controller) {
268
273
  if (!(chunk instanceof Int16Array)) {
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new Promise<void>((resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(\n new APITimeoutError({\n message: 'WebSocket connection timeout',\n options: { retryable: false },\n }),\n );\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('unexpected-response', (_req, res) => {\n clearTimeout(timeout);\n ws.terminate();\n const statusCode = res.statusCode ?? -1;\n reject(\n new APIStatusError({\n message: `WebSocket connection rejected with status ${statusCode}`,\n options: { statusCode, retryable: false },\n }),\n );\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));\n });\n });\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n outputController?.error(\n new APIConnectionError({ message: `WebSocket error: ${err.message}` }),\n );\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<void> {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new APIStatusError({\n message: `LiveKit Adaptive Interruption error: ${message.message}`,\n options: { statusCode: message.code ?? -1 },\n }),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new APIConnectionError({ message: 'WebSocket not connected' });\n }\n\n const state = getState();\n const createdAt = Math.floor(performance.now());\n\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n ws.send(combined);\n onRequestSent?.();\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection();\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n if (options.timeout > 0) {\n const now = performance.now();\n for (const [, entry] of state.cache.entries()) {\n if (entry.totalDurationInS !== 0) continue;\n if (now - entry.createdAt > options.timeout) {\n controller.error(\n new APIStatusError({\n message: `interruption inference timed out after ${((now - entry.createdAt) / 1000).toFixed(1)}s (ws)`,\n options: { statusCode: 408, retryable: false },\n }),\n );\n return;\n }\n break;\n }\n }\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n controller.error(err);\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":"AAGA,SAAS,uBAAuB;AAChC,OAAO,eAAe;AACtB,SAAS,SAAS;AAClB,SAAS,oBAAoB,gBAAgB,uBAAuB;AACpE,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAClC,SAAS,8BAA8B;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,EAAE,mBAAmB,QAAQ;AAAA,EACnD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,EAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,EAAE,OAAO;AAAA,IAClB,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb;AAAA,QACE,IAAI,gBAAgB;AAAA,UAClB,SAAS;AAAA,UACT,SAAS,EAAE,WAAW,MAAM;AAAA,QAC9B,CAAC;AAAA,MACH;AAAA,IACF,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,uBAAuB,CAAC,MAAM,QAAQ;AAC5C,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,YAAM,aAAa,IAAI,cAAc;AACrC;AAAA,QACE,IAAI,eAAe;AAAA,UACjB,SAAS,6CAA6C,UAAU;AAAA,UAChE,SAAS,EAAE,YAAY,WAAW,MAAM;AAAA,QAC1C,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,IAAI,mBAAmB,EAAE,SAAS,+BAA+B,IAAI,OAAO,GAAG,CAAC,CAAC;AAAA,IAC1F,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,SAAS,IAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,2DAAkB;AAAA,QAChB,IAAI,mBAAmB,EAAE,SAAS,oBAAoB,IAAI,OAAO,GAAG,CAAC;AAAA;AAAA,IAEzE,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAU,KAAM;AAE5C,SAAK,MAAM,iBAAiB,OAAO;AACnC,wBAAoB,EAAE;AAEtB,UAAM,mBAAmB,KAAK,UAAU;AAAA,MACtC,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa,QAAQ;AAAA,QACrB,cAAc;AAAA,QACd,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB,UAAU;AAAA,MACZ;AAAA,IACF,CAAC;AACD,OAAG,KAAK,gBAAgB;AAAA,EAC1B;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,YAAY,KAAK,IAAI;AAAA,YACrB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI,eAAe;AAAA,YACjB,SAAS,wCAAwC,QAAQ,OAAO;AAAA,YAChE,SAAS,EAAE,YAAY,QAAQ,QAAQ,GAAG;AAAA,UAC5C,CAAC;AAAA;AAEH;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAU,MAAM;AAC3C,YAAM,IAAI,mBAAmB,EAAE,SAAS,0BAA0B,CAAC;AAAA,IACrE;AAEA,UAAM,QAAQ,SAAS;AACvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAE9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uBAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAEjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,OAAG,KAAK,QAAQ;AAChB;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI,QAAQ,UAAU,GAAG;AACvB,gBAAM,MAAM,YAAY,IAAI;AAC5B,qBAAW,CAAC,EAAE,KAAK,KAAK,MAAM,MAAM,QAAQ,GAAG;AAC7C,gBAAI,MAAM,qBAAqB,EAAG;AAClC,gBAAI,MAAM,MAAM,YAAY,QAAQ,SAAS;AAC3C,yBAAW;AAAA,gBACT,IAAI,eAAe;AAAA,kBACjB,SAAS,4CAA4C,MAAM,MAAM,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,kBAC9F,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,gBAC/C,CAAC;AAAA,cACH;AACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF;AAEA,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":[]}
1
+ {"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Throws } from '@livekit/throws-transformer/throws';\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport TypedPromise from '../../typed_promise.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n sampleRate: number;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface WsTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n z.object({\n type: z.literal(MSG_SESSION_CREATED),\n }),\n z.object({\n type: z.literal(MSG_SESSION_CLOSED),\n }),\n z.object({\n type: z.literal(MSG_INTERRUPTION_DETECTED),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n }),\n z.object({\n type: z.literal(MSG_INFERENCE_DONE),\n created_at: z.number(),\n probabilities: z.array(z.number()).default([]),\n prediction_duration: z.number().default(0),\n is_bargein: z.boolean().optional(),\n }),\n z.object({\n type: z.literal(MSG_ERROR),\n message: z.string(),\n code: z.number().optional(),\n session_id: z.string().optional(),\n }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(\n options: WsTransportOptions,\n): Promise<Throws<WebSocket, APIStatusError | APITimeoutError | APIConnectionError>> {\n const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n const token = await createAccessToken(options.apiKey, options.apiSecret);\n const url = `${baseUrl}/bargein`;\n\n const ws = new WebSocket(url, {\n headers: { Authorization: `Bearer ${token}` },\n });\n\n await new TypedPromise<void, APIStatusError | APITimeoutError | APIConnectionError>(\n (resolve, reject) => {\n const timeout = setTimeout(() => {\n ws.terminate();\n reject(\n new APITimeoutError({\n message: 'WebSocket connection timeout',\n options: { retryable: false },\n }),\n );\n }, options.timeout);\n ws.once('open', () => {\n clearTimeout(timeout);\n resolve();\n });\n ws.once('unexpected-response', (_req, res) => {\n clearTimeout(timeout);\n ws.terminate();\n const statusCode = res.statusCode ?? -1;\n reject(\n new APIStatusError({\n message: `WebSocket connection rejected with status ${statusCode}`,\n options: { statusCode, retryable: false },\n }),\n );\n });\n ws.once('error', (err: Error) => {\n clearTimeout(timeout);\n ws.terminate();\n reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));\n });\n },\n );\n\n return ws;\n}\n\nexport interface WsTransportResult {\n transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n options: WsTransportOptions,\n getState: () => WsTransportState,\n setState: (partial: Partial<WsTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n onRequestSent?: () => void,\n getAndResetNumRequests?: () => number,\n): WsTransportResult {\n const logger = log();\n let ws: WebSocket | null = null;\n let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n function setupMessageHandler(socket: WebSocket): void {\n socket.on('message', (data: WebSocket.Data) => {\n try {\n const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n handleMessage(message);\n } catch {\n logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n }\n });\n\n socket.on('error', (err: Error) => {\n outputController?.error(\n new APIConnectionError({ message: `WebSocket error: ${err.message}` }),\n );\n });\n\n socket.on('close', (code: number, reason: Buffer) => {\n logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n });\n }\n\n async function ensureConnection(): Promise<\n Throws<void, APIStatusError | APITimeoutError | APIConnectionError>\n > {\n if (ws && ws.readyState === WebSocket.OPEN) return;\n\n ws = await connectWebSocket(options);\n setupMessageHandler(ws);\n\n const sessionCreateMsg = JSON.stringify({\n type: MSG_SESSION_CREATE,\n settings: {\n sample_rate: options.sampleRate,\n num_channels: 1,\n threshold: options.threshold,\n min_frames: options.minFrames,\n encoding: 's16le',\n },\n });\n ws.send(sessionCreateMsg);\n }\n\n function handleMessage(message: WsMessage): void {\n const state = getState();\n\n switch (message.type) {\n case MSG_SESSION_CREATED:\n logger.debug('WebSocket session created');\n break;\n\n case MSG_INTERRUPTION_DETECTED: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n probabilities: message.probabilities,\n isInterruption: true,\n predictionDurationInS: message.prediction_duration,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n\n logger.debug(\n {\n totalDuration: entry.totalDurationInS,\n predictionDuration: entry.predictionDurationInS,\n detectionDelay: entry.detectionDelayInS,\n probability: entry.probability,\n },\n 'interruption detected',\n );\n\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n isInterruption: true,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n overlapStartedAt: overlapSpeechStartedAt,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n\n outputController?.enqueue(event);\n setState({ overlapSpeechStarted: false });\n }\n break;\n }\n\n case MSG_INFERENCE_DONE: {\n const createdAt = message.created_at;\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n const existing = state.cache.get(createdAt);\n const totalDurationInS =\n existing?.requestStartedAt !== undefined\n ? (performance.now() - existing.requestStartedAt) / 1000\n : (performance.now() - createdAt) / 1000;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n speechInput: existing?.speechInput,\n requestStartedAt: existing?.requestStartedAt,\n totalDurationInS,\n predictionDurationInS: message.prediction_duration,\n probabilities: message.probabilities,\n isInterruption: message.is_bargein ?? false,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n },\n );\n\n logger.debug(\n {\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n },\n 'interruption inference done',\n );\n }\n break;\n }\n\n case MSG_SESSION_CLOSED:\n logger.debug('WebSocket session closed');\n break;\n\n case MSG_ERROR:\n outputController?.error(\n new APIStatusError({\n message: `LiveKit Adaptive Interruption error: ${message.message}`,\n options: { statusCode: message.code ?? -1 },\n }),\n );\n break;\n }\n }\n\n function sendAudioData(audioSlice: Int16Array): void {\n if (!ws || ws.readyState !== WebSocket.OPEN) {\n throw new APIConnectionError({ message: 'WebSocket not connected' });\n }\n\n const state = getState();\n const createdAt = Math.floor(performance.now());\n\n state.cache.set(\n createdAt,\n new InterruptionCacheEntry({\n createdAt,\n requestStartedAt: performance.now(),\n speechInput: audioSlice,\n }),\n );\n\n const header = new ArrayBuffer(8);\n const view = new DataView(header);\n view.setUint32(0, createdAt >>> 0, true);\n view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n const audioBytes = new Uint8Array(\n audioSlice.buffer,\n audioSlice.byteOffset,\n audioSlice.byteLength,\n );\n const combined = new Uint8Array(8 + audioBytes.length);\n combined.set(new Uint8Array(header), 0);\n combined.set(audioBytes, 8);\n\n ws.send(combined);\n onRequestSent?.();\n }\n\n function close(): void {\n if (ws?.readyState === WebSocket.OPEN) {\n const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n try {\n ws.send(closeMsg);\n } catch (e: unknown) {\n logger.error(e, 'failed to send close message');\n }\n }\n ws?.close(1000); // signal normal websocket closure\n ws = null;\n }\n\n /**\n * Reconnect the WebSocket with updated options.\n * This is called when options are updated via updateOptions().\n */\n async function reconnect(): Promise<void> {\n close();\n }\n\n const transport = new TransformStream<\n Int16Array | OverlappingSpeechEvent,\n OverlappingSpeechEvent\n >(\n {\n async start(controller) {\n outputController = controller;\n await ensureConnection().catch((e) => {\n controller.error(e);\n });\n },\n\n transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n // Only forwards buffered audio while overlap speech is actively on.\n const state = getState();\n if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n if (options.timeout > 0) {\n const now = performance.now();\n for (const [, entry] of state.cache.entries()) {\n if (entry.totalDurationInS !== 0) continue;\n if (now - entry.createdAt > options.timeout) {\n controller.error(\n new APIStatusError({\n message: `interruption inference timed out after ${((now - entry.createdAt) / 1000).toFixed(1)}s (ws)`,\n options: { statusCode: 408, retryable: false },\n }),\n );\n return;\n }\n break;\n }\n }\n\n try {\n sendAudioData(chunk);\n } catch (err) {\n controller.error(err);\n }\n },\n\n flush() {\n close();\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n\n return { transport, reconnect };\n}\n"],"mappings":"AAIA,SAAS,uBAAuB;AAChC,OAAO,eAAe;AACtB,SAAS,SAAS;AAClB,SAAS,oBAAoB,gBAAgB,uBAAuB;AACpE,SAAS,WAAW;AACpB,OAAO,kBAAkB;AACzB,SAAS,yBAAyB;AAClC,SAAS,8BAA8B;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,EAAE,mBAAmB,QAAQ;AAAA,EACnD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,EAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,EAAE,OAAO;AAAA,IAClB,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBACb,SACmF;AACnF,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI;AAAA,IACR,CAAC,SAAS,WAAW;AACnB,YAAM,UAAU,WAAW,MAAM;AAC/B,WAAG,UAAU;AACb;AAAA,UACE,IAAI,gBAAgB;AAAA,YAClB,SAAS;AAAA,YACT,SAAS,EAAE,WAAW,MAAM;AAAA,UAC9B,CAAC;AAAA,QACH;AAAA,MACF,GAAG,QAAQ,OAAO;AAClB,SAAG,KAAK,QAAQ,MAAM;AACpB,qBAAa,OAAO;AACpB,gBAAQ;AAAA,MACV,CAAC;AACD,SAAG,KAAK,uBAAuB,CAAC,MAAM,QAAQ;AAC5C,qBAAa,OAAO;AACpB,WAAG,UAAU;AACb,cAAM,aAAa,IAAI,cAAc;AACrC;AAAA,UACE,IAAI,eAAe;AAAA,YACjB,SAAS,6CAA6C,UAAU;AAAA,YAChE,SAAS,EAAE,YAAY,WAAW,MAAM;AAAA,UAC1C,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AACD,SAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,qBAAa,OAAO;AACpB,WAAG,UAAU;AACb,eAAO,IAAI,mBAAmB,EAAE,SAAS,+BAA+B,IAAI,OAAO,GAAG,CAAC,CAAC;AAAA,MAC1F,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,SAAS,IAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,2DAAkB;AAAA,QAChB,IAAI,mBAAmB,EAAE,SAAS,oBAAoB,IAAI,OAAO,GAAG,CAAC;AAAA;AAAA,IAEzE,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAEb;AACA,QAAI,MAAM,GAAG,eAAe,UAAU,KAAM;AAE5C,SAAK,MAAM,iBAAiB,OAAO;AACnC,wBAAoB,EAAE;AAEtB,UAAM,mBAAmB,KAAK,UAAU;AAAA,MACtC,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa,QAAQ;AAAA,QACrB,cAAc;AAAA,QACd,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB,UAAU;AAAA,MACZ;AAAA,IACF,CAAC;AACD,OAAG,KAAK,gBAAgB;AAAA,EAC1B;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,YAAY,KAAK,IAAI;AAAA,YACrB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI,eAAe;AAAA,YACjB,SAAS,wCAAwC,QAAQ,OAAO;AAAA,YAChE,SAAS,EAAE,YAAY,QAAQ,QAAQ,GAAG;AAAA,UAC5C,CAAC;AAAA;AAEH;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAU,MAAM;AAC3C,YAAM,IAAI,mBAAmB,EAAE,SAAS,0BAA0B,CAAC;AAAA,IACrE;AAEA,UAAM,QAAQ,SAAS;AACvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAE9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uBAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAEjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,OAAG,KAAK,QAAQ;AAChB;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM;AACpC,qBAAW,MAAM,CAAC;AAAA,QACpB,CAAC;AAAA,MACH;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI,QAAQ,UAAU,GAAG;AACvB,gBAAM,MAAM,YAAY,IAAI;AAC5B,qBAAW,CAAC,EAAE,KAAK,KAAK,MAAM,MAAM,QAAQ,GAAG;AAC7C,gBAAI,MAAM,qBAAqB,EAAG;AAClC,gBAAI,MAAM,MAAM,YAAY,QAAQ,SAAS;AAC3C,yBAAW;AAAA,gBACT,IAAI,eAAe;AAAA,kBACjB,SAAS,4CAA4C,MAAM,MAAM,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,kBAC9F,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,gBAC/C,CAAC;AAAA,cACH;AACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF;AAEA,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":[]}
@@ -356,13 +356,18 @@ class SynthesizeStream extends import_tts.SynthesizeStream {
356
356
  };
357
357
  const createRecvTask = async (signal) => {
358
358
  let currentSessionId = null;
359
+ const recvTimeoutMs = this.connOptions.timeoutMs;
359
360
  const bstream = new import_audio.AudioByteStream(this.opts.sampleRate, NUM_CHANNELS);
360
361
  const serverEventStream = eventChannel.stream();
361
362
  const reader = serverEventStream.getReader();
362
363
  try {
363
364
  await inputSentEvent.wait();
364
365
  while (!this.closed && !signal.aborted) {
365
- const result = await reader.read();
366
+ const result = await (0, import_utils.waitUntilTimeout)(
367
+ reader.read(),
368
+ recvTimeoutMs,
369
+ () => new import_exceptions.APITimeoutError({ message: "TTS recv idle timeout" })
370
+ );
366
371
  if (signal.aborted) return;
367
372
  if (result.done) return;
368
373
  const serverEvent = result.value;
@@ -406,6 +411,14 @@ class SynthesizeStream extends import_tts.SynthesizeStream {
406
411
  break;
407
412
  }
408
413
  }
414
+ } catch (e) {
415
+ if (e instanceof import_exceptions.APITimeoutError) {
416
+ this.#logger.warn("TTS recv task timed out waiting for server message");
417
+ await resourceCleanup();
418
+ completionFuture.reject(e);
419
+ return;
420
+ }
421
+ throw e;
409
422
  } finally {
410
423
  reader.releaseLock();
411
424
  try {
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/inference/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { WebSocket } from 'ws';\nimport { APIError, APIStatusError } from '../_exceptions.js';\nimport { AudioByteStream } from '../audio.js';\nimport { ConnectionPool } from '../connection_pool.js';\nimport { type LanguageCode, normalizeLanguage } from '../language.js';\nimport { log } from '../log.js';\nimport { createStreamChannel } from '../stream/stream_channel.js';\nimport { basic as tokenizeBasic } from '../tokenize/index.js';\nimport type { ChunkedStream } from '../tts/index.js';\nimport { SynthesizeStream as BaseSynthesizeStream, TTS as BaseTTS } from '../tts/index.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { Event, Future, Task, cancelAndWait, combineSignals, shortuuid } from '../utils.js';\nimport {\n type TtsClientEvent,\n type TtsServerEvent,\n ttsClientEventSchema,\n ttsServerEventSchema,\n} from './api_protos.js';\nimport { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js';\n\nexport type CartesiaModels =\n | 'cartesia/sonic-3'\n | 'cartesia/sonic-2'\n | 'cartesia/sonic-turbo'\n | 'cartesia/sonic';\n\nexport type DeepgramTTSModels = 'deepgram/aura' | 'deepgram/aura-2';\n\nexport type ElevenlabsModels =\n | 'elevenlabs/eleven_flash_v2'\n | 'elevenlabs/eleven_flash_v2_5'\n | 'elevenlabs/eleven_turbo_v2'\n | 'elevenlabs/eleven_turbo_v2_5'\n | 'elevenlabs/eleven_multilingual_v2';\n\nexport type InworldModels =\n | 'inworld/inworld-tts-1.5-max'\n | 'inworld/inworld-tts-1.5-mini'\n | 'inworld/inworld-tts-1-max'\n | 'inworld/inworld-tts-1';\n\nexport type RimeModels = 'rime/arcana' | 'rime/mistv2';\n\nexport interface CartesiaOptions {\n /** Maximum duration of audio in seconds. */\n duration?: number;\n /** Speech speed. Default: not specified. */\n speed?: 'slow' | 'normal' | 'fast';\n}\n\nexport interface ElevenlabsOptions {\n /** Inactivity timeout in seconds. Default: 60. */\n inactivity_timeout?: number;\n /** Text normalization mode. Default: \"auto\". */\n apply_text_normalization?: 'auto' | 'off' | 'on';\n}\n\nexport interface DeepgramTTSOptions {}\n\nexport interface RimeOptions {}\n\nexport interface InworldOptions {\n /** Controls how fast the voice speaks. 1.0 is normal speed, 0.5 is half, 1.5 is 1.5x. Default: 1.0. */\n speaking_rate?: number;\n /** Controls randomness in the output. Recommended between 0.6 and 1.1. Default: 1.1. */\n temperature?: number;\n /** Controls text normalization. \"ON\" expands numbers, dates, abbreviations. \"OFF\" reads text as written. Default: \"ON\". */\n text_normalization?: 'ON' | 'OFF';\n}\n\ntype _TTSModels =\n | CartesiaModels\n | DeepgramTTSModels\n | ElevenlabsModels\n | RimeModels\n | InworldModels;\n\nexport type TTSModels =\n | CartesiaModels\n | DeepgramTTSModels\n | ElevenlabsModels\n | RimeModels\n | InworldModels\n | AnyString;\n\nexport type ModelWithVoice = `${_TTSModels}:${string}` | TTSModels;\n\nexport type TTSOptions<TModel extends TTSModels> = TModel extends CartesiaModels\n ? CartesiaOptions\n : TModel extends DeepgramTTSModels\n ? DeepgramTTSOptions\n : TModel extends ElevenlabsModels\n ? ElevenlabsOptions\n : TModel extends RimeModels\n ? RimeOptions\n : TModel extends InworldModels\n ? InworldOptions\n : Record<string, unknown>;\n\n/** Parse a model string into [model, voice]. Voice is undefined if not specified. */\nexport function parseTTSModelString(model: string): [string, string | undefined] {\n const idx = model.lastIndexOf(':');\n if (idx !== -1) {\n return [model.slice(0, idx), model.slice(idx + 1)];\n }\n return [model, undefined];\n}\n\n/** A fallback model with optional extra configuration. Extra fields are passed through to the provider. */\nexport interface TTSFallbackModel {\n /** Model name (e.g. \"cartesia/sonic\", \"elevenlabs/eleven_flash_v2\", \"rime/arcana\"). */\n model: string;\n /** Voice to use for the model. */\n voice: string;\n /** Extra configuration for the model. */\n extraKwargs?: Record<string, unknown>;\n}\n\nexport type TTSFallbackModelType = TTSFallbackModel | string;\n\n/** Normalize a single or list of FallbackModelType into TTSFallbackModel[]. */\nexport function normalizeTTSFallback(\n fallback: TTSFallbackModelType | TTSFallbackModelType[],\n): TTSFallbackModel[] {\n const makeFallback = (model: TTSFallbackModelType): TTSFallbackModel => {\n if (typeof model === 'string') {\n const [name, voice] = parseTTSModelString(model);\n return { model: name, voice: voice ?? '' };\n }\n return model;\n };\n\n if (Array.isArray(fallback)) {\n return fallback.map(makeFallback);\n }\n return [makeFallback(fallback)];\n}\n\ntype TTSEncoding = 'pcm_s16le';\n\nconst DEFAULT_ENCODING: TTSEncoding = 'pcm_s16le';\nconst DEFAULT_SAMPLE_RATE = 16000;\nconst NUM_CHANNELS = 1;\nconst DEFAULT_LANGUAGE = 'en';\n\nexport interface InferenceTTSOptions<TModel extends TTSModels> {\n model?: TModel;\n voice?: string;\n language?: LanguageCode;\n encoding: TTSEncoding;\n sampleRate: number;\n baseURL: string;\n apiKey: string;\n apiSecret: string;\n modelOptions: TTSOptions<TModel>;\n fallback?: TTSFallbackModel[];\n connOptions?: APIConnectOptions;\n}\n\n/**\n * Livekit Cloud Inference TTS\n */\nexport class TTS<TModel extends TTSModels> extends BaseTTS {\n private opts: InferenceTTSOptions<TModel>;\n private streams: Set<SynthesizeStream<TModel>> = new Set();\n pool: ConnectionPool<WebSocket>;\n\n #logger = log();\n\n constructor(opts: {\n model: TModel;\n voice?: string;\n language?: string;\n baseURL?: string;\n encoding?: TTSEncoding;\n sampleRate?: number;\n apiKey?: string;\n apiSecret?: string;\n modelOptions?: TTSOptions<TModel>;\n fallback?: TTSFallbackModelType | TTSFallbackModelType[];\n connOptions?: APIConnectOptions;\n }) {\n const sampleRate = opts?.sampleRate ?? DEFAULT_SAMPLE_RATE;\n super(sampleRate, 1, { streaming: true });\n\n const {\n model,\n voice,\n language = DEFAULT_LANGUAGE,\n baseURL,\n encoding = DEFAULT_ENCODING,\n apiKey,\n apiSecret,\n modelOptions = {} as TTSOptions<TModel>,\n fallback,\n connOptions,\n } = opts || {};\n\n const lkBaseURL = baseURL || getDefaultInferenceUrl();\n const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;\n if (!lkApiKey) {\n throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');\n }\n\n const lkApiSecret =\n apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;\n if (!lkApiSecret) {\n throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');\n }\n\n // read voice id from the model if provided: \"provider/model:voice_id\"\n let nextModel = model;\n let nextVoice = voice;\n if (typeof nextModel === 'string') {\n const idx = nextModel.lastIndexOf(':');\n if (idx !== -1) {\n const voiceFromModel = nextModel.slice(idx + 1);\n if (nextVoice && nextVoice !== voiceFromModel) {\n this.#logger.warn(\n '`voice` is provided via both argument and model, using the one from the argument',\n { voice: nextVoice, model: nextModel },\n );\n } else {\n nextVoice = voiceFromModel;\n }\n nextModel = nextModel.slice(0, idx) as TModel;\n }\n }\n\n const normalizedFallback = fallback ? normalizeTTSFallback(fallback) : undefined;\n\n this.opts = {\n model: nextModel,\n voice: nextVoice,\n language: normalizeLanguage(language),\n encoding,\n sampleRate,\n baseURL: lkBaseURL,\n apiKey: lkApiKey,\n apiSecret: lkApiSecret,\n modelOptions,\n fallback: normalizedFallback,\n connOptions: connOptions ?? DEFAULT_API_CONNECT_OPTIONS,\n };\n\n // Initialize connection pool\n this.pool = new ConnectionPool<WebSocket>({\n connectCb: (timeout) => this.connectWs(timeout),\n closeCb: (ws) => this.closeWs(ws),\n maxSessionDuration: 300_000,\n markRefreshedOnGet: true,\n connectTimeout: 10_000, // 10 seconds default\n });\n }\n\n get label() {\n return 'inference.TTS';\n }\n\n get model(): string {\n return this.opts.model ?? 'unknown';\n }\n\n get provider(): string {\n return 'livekit';\n }\n\n static fromModelString(modelString: string): TTS<AnyString> {\n const [model, voice] = parseTTSModelString(modelString);\n return new TTS({ model, voice: voice || undefined });\n }\n\n updateOptions(opts: Partial<Pick<InferenceTTSOptions<TModel>, 'model' | 'voice' | 'language'>>) {\n this.opts = {\n ...this.opts,\n ...opts,\n language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,\n };\n for (const stream of this.streams) {\n stream.updateOptions(this.opts);\n }\n }\n\n synthesize(_: string): ChunkedStream {\n throw new Error('ChunkedStream is not implemented');\n }\n\n stream(options?: { connOptions?: APIConnectOptions }): SynthesizeStream<TModel> {\n const { connOptions = this.opts.connOptions ?? DEFAULT_API_CONNECT_OPTIONS } = options || {};\n const stream = new SynthesizeStream(this, { ...this.opts }, connOptions);\n this.streams.add(stream);\n return stream;\n }\n\n async connectWs(timeout: number): Promise<WebSocket> {\n let baseURL = this.opts.baseURL;\n if (baseURL.startsWith('http://') || baseURL.startsWith('https://')) {\n baseURL = baseURL.replace('http', 'ws');\n }\n\n const token = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);\n const url = `${baseURL}/tts`;\n const headers = { Authorization: `Bearer ${token}` } as Record<string, string>;\n\n const params = {\n type: 'session.create',\n sample_rate: String(this.opts.sampleRate),\n encoding: this.opts.encoding,\n extra: this.opts.modelOptions,\n } as Record<string, unknown>;\n\n if (this.opts.voice) (params as Record<string, unknown>).voice = this.opts.voice;\n if (this.opts.model) (params as Record<string, unknown>).model = this.opts.model;\n if (this.opts.language) (params as Record<string, unknown>).language = this.opts.language;\n\n if (this.opts.fallback?.length) {\n params.fallback = {\n models: this.opts.fallback.map((m) => ({\n model: m.model,\n voice: m.voice,\n extra: m.extraKwargs ?? {},\n })),\n };\n }\n\n if (this.opts.connOptions) {\n params.connection = {\n timeout: this.opts.connOptions.timeoutMs / 1000,\n retries: this.opts.connOptions.maxRetry,\n };\n }\n\n this.#logger.debug({ url }, 'inference.TTS creating new websocket connection (pool miss)');\n const socket = await connectWs(url, headers, timeout);\n socket.send(JSON.stringify(params));\n return socket;\n }\n\n async closeWs(ws: WebSocket) {\n await ws.close();\n }\n\n prewarm(): void {\n this.pool.prewarm();\n }\n\n async close() {\n for (const stream of this.streams) {\n await stream.close();\n }\n this.streams.clear();\n await this.pool.close();\n }\n}\n\nexport class SynthesizeStream<TModel extends TTSModels> extends BaseSynthesizeStream {\n private opts: InferenceTTSOptions<TModel>;\n private tts: TTS<TModel>;\n\n #logger = log();\n\n constructor(tts: TTS<TModel>, opts: InferenceTTSOptions<TModel>, connOptions: APIConnectOptions) {\n super(tts, connOptions);\n this.opts = opts;\n this.tts = tts;\n }\n\n get label() {\n return 'inference.SynthesizeStream';\n }\n\n updateOptions(opts: Partial<Pick<InferenceTTSOptions<TModel>, 'model' | 'voice' | 'language'>>) {\n this.opts = {\n ...this.opts,\n ...opts,\n language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,\n };\n }\n\n protected async run(): Promise<void> {\n let closing = false;\n let lastFrame: AudioFrame | undefined;\n\n const sendTokenizerStream = new tokenizeBasic.SentenceTokenizer().stream();\n const eventChannel = createStreamChannel<TtsServerEvent>();\n const requestId = shortuuid('tts_request_');\n const inputSentEvent = new Event();\n\n // Signal for protocol-driven completion (when 'done' message is received)\n const completionFuture = new Future<void>();\n\n const resourceCleanup = async () => {\n if (closing) return;\n closing = true;\n sendTokenizerStream.close();\n // close() returns a promise; don't leak it\n await eventChannel.close();\n };\n\n const sendClientEvent = async (event: TtsClientEvent, ws: WebSocket, signal: AbortSignal) => {\n // Don't send events to a closed WebSocket or aborted controller\n if (signal.aborted || closing) return;\n\n const validatedEvent = await ttsClientEventSchema.parseAsync(event);\n if (ws.readyState !== WebSocket.OPEN) {\n this.#logger.warn('Trying to send client TTS event to a closed WebSocket');\n return;\n }\n ws.send(JSON.stringify(validatedEvent));\n };\n\n const sendLastFrame = (segmentId: string, final: boolean) => {\n if (lastFrame) {\n this.queue.put({ requestId, segmentId, frame: lastFrame, final });\n lastFrame = undefined;\n }\n };\n\n const createInputTask = async (signal: AbortSignal) => {\n for await (const data of this.input) {\n if (signal.aborted || closing) break;\n if (data === SynthesizeStream.FLUSH_SENTINEL) {\n sendTokenizerStream.flush();\n continue;\n }\n sendTokenizerStream.pushText(data);\n }\n // Only call endInput if the stream hasn't been closed by cleanup\n if (!closing) {\n sendTokenizerStream.endInput();\n }\n };\n\n const createSentenceStreamTask = async (ws: WebSocket, signal: AbortSignal) => {\n for await (const ev of sendTokenizerStream) {\n if (signal.aborted || closing) break;\n\n await sendClientEvent(\n {\n type: 'input_transcript',\n transcript: ev.token + ' ',\n },\n ws,\n signal,\n );\n inputSentEvent.set();\n }\n\n await sendClientEvent({ type: 'session.flush' }, ws, signal);\n // needed in case empty input is sent\n inputSentEvent.set();\n };\n\n // Handles WebSocket message routing and error handling\n // Completes based on protocol messages, NOT on ws.close()\n const createWsListenerTask = async (ws: WebSocket, signal: AbortSignal) => {\n const onMessage = (data: Buffer) => {\n try {\n const eventJson = JSON.parse(data.toString()) as Record<string, unknown>;\n const validatedEvent = ttsServerEventSchema.parse(eventJson);\n // writer.write returns a promise; avoid unhandled rejections if stream is closed\n void eventChannel.write(validatedEvent).catch((error) => {\n this.#logger.debug(\n { error },\n 'Failed writing TTS event to stream channel (likely closed)',\n );\n });\n } catch (e) {\n this.#logger.error({ error: e }, 'Error parsing WebSocket message');\n }\n };\n\n const onError = (e: Error) => {\n this.#logger.error({ error: e }, 'WebSocket error');\n void resourceCleanup();\n try {\n // If the ws is misbehaving, hard-stop it immediately to avoid buffering.\n ws.terminate?.();\n } catch {\n // ignore\n }\n // Ensure this ws is not reused\n this.tts.pool.remove(ws);\n completionFuture.reject(e);\n };\n\n const onClose = () => {\n // WebSocket closed unexpectedly (not by us)\n if (!closing) {\n this.#logger.error('WebSocket closed unexpectedly');\n void resourceCleanup();\n // Ensure this ws is not reused\n this.tts.pool.remove(ws);\n completionFuture.reject(\n new APIStatusError({\n message: 'Gateway connection closed unexpectedly',\n options: { requestId },\n }),\n );\n }\n };\n\n const onAbort = () => {\n void resourceCleanup();\n try {\n // On interruption/abort, close the websocket immediately so the server stops streaming\n // and the ws library doesn't buffer unread frames in memory.\n ws.terminate?.();\n } catch {\n // ignore\n }\n this.tts.pool.remove(ws);\n inputSentEvent.set();\n completionFuture.resolve();\n };\n\n // Attach listeners\n ws.on('message', onMessage);\n ws.on('error', onError);\n ws.on('close', onClose);\n signal.addEventListener('abort', onAbort);\n\n try {\n // Wait for protocol-driven completion or error\n await completionFuture.await;\n } finally {\n // IMPORTANT: Remove listeners so connection can be reused\n ws.off('message', onMessage);\n ws.off('error', onError);\n ws.off('close', onClose);\n signal.removeEventListener('abort', onAbort);\n }\n };\n\n const createRecvTask = async (signal: AbortSignal) => {\n let currentSessionId: string | null = null;\n\n const bstream = new AudioByteStream(this.opts.sampleRate, NUM_CHANNELS);\n const serverEventStream = eventChannel.stream();\n const reader = serverEventStream.getReader();\n\n try {\n await inputSentEvent.wait();\n\n while (!this.closed && !signal.aborted) {\n const result = await reader.read();\n if (signal.aborted) return;\n if (result.done) return;\n\n const serverEvent = result.value;\n switch (serverEvent.type) {\n case 'session.created':\n currentSessionId = serverEvent.session_id;\n break;\n case 'output_audio':\n const base64Data = new Int8Array(Buffer.from(serverEvent.audio, 'base64'));\n for (const frame of bstream.write(base64Data.buffer)) {\n sendLastFrame(currentSessionId!, false);\n lastFrame = frame;\n }\n break;\n case 'done':\n for (const frame of bstream.flush()) {\n sendLastFrame(currentSessionId!, false);\n lastFrame = frame;\n }\n sendLastFrame(currentSessionId!, true);\n this.queue.put(SynthesizeStream.END_OF_STREAM);\n await resourceCleanup();\n completionFuture.resolve();\n return;\n case 'session.closed':\n await resourceCleanup();\n completionFuture.resolve();\n return;\n case 'error':\n this.#logger.error(\n { serverEvent },\n 'Received error message from LiveKit TTS WebSocket',\n );\n await resourceCleanup();\n completionFuture.reject(\n new APIError(`LiveKit TTS returned error: ${serverEvent.message}`),\n );\n return;\n default:\n this.#logger.warn('Unexpected message %s', serverEvent);\n break;\n }\n }\n } finally {\n reader.releaseLock();\n try {\n await serverEventStream.cancel();\n } catch (e) {\n this.#logger.debug('Error cancelling serverEventStream (may already be cancelled):', e);\n }\n }\n };\n\n try {\n await this.tts.pool.withConnection(\n async (ws: WebSocket) => {\n try {\n // IMPORTANT: don't cancel the stream's controller on normal completion,\n // otherwise the pool will remove+close the ws and every run becomes a pool miss.\n const runController = new AbortController();\n const onStreamAbort = () => runController.abort(this.abortController.signal.reason);\n this.abortController.signal.addEventListener('abort', onStreamAbort, { once: true });\n\n const tasks = [\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createInputTask(combined);\n },\n undefined,\n 'inference-tts-input',\n ),\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createSentenceStreamTask(ws, combined);\n },\n undefined,\n 'inference-tts-sentence',\n ),\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createWsListenerTask(ws, combined);\n },\n undefined,\n 'inference-tts-ws-listener',\n ),\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createRecvTask(combined);\n },\n undefined,\n 'inference-tts-recv',\n ),\n ];\n\n try {\n await Promise.all(tasks.map((t) => t.result));\n } finally {\n // Mirror python finally: unblock recv and cancel all tasks.\n inputSentEvent.set();\n await resourceCleanup();\n await cancelAndWait(tasks, 5000);\n this.abortController.signal.removeEventListener('abort', onStreamAbort);\n }\n } catch (e) {\n // If aborted, don't throw - let cleanup handle it\n if (e instanceof Error && e.name === 'AbortError') {\n return;\n }\n throw e;\n }\n },\n {\n timeout: this.connOptions.timeoutMs,\n },\n );\n } catch (e) {\n // Handle connection errors\n if (e instanceof Error && e.name === 'AbortError') {\n // Abort is expected during normal shutdown\n return;\n }\n throw e;\n } finally {\n // Ensure cleanup always runs (and don't leak the promise)\n await resourceCleanup();\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,gBAA0B;AAC1B,wBAAyC;AACzC,mBAAgC;AAChC,6BAA+B;AAC/B,sBAAqD;AACrD,iBAAoB;AACpB,4BAAoC;AACpC,sBAAuC;AAEvC,iBAAyE;AACzE,mBAAoE;AACpE,mBAA8E;AAC9E,wBAKO;AACP,IAAAA,gBAAqF;AAkF9E,SAAS,oBAAoB,OAA6C;AAC/E,QAAM,MAAM,MAAM,YAAY,GAAG;AACjC,MAAI,QAAQ,IAAI;AACd,WAAO,CAAC,MAAM,MAAM,GAAG,GAAG,GAAG,MAAM,MAAM,MAAM,CAAC,CAAC;AAAA,EACnD;AACA,SAAO,CAAC,OAAO,MAAS;AAC1B;AAeO,SAAS,qBACd,UACoB;AACpB,QAAM,eAAe,CAAC,UAAkD;AACtE,QAAI,OAAO,UAAU,UAAU;AAC7B,YAAM,CAAC,MAAM,KAAK,IAAI,oBAAoB,KAAK;AAC/C,aAAO,EAAE,OAAO,MAAM,OAAO,SAAS,GAAG;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,QAAQ,GAAG;AAC3B,WAAO,SAAS,IAAI,YAAY;AAAA,EAClC;AACA,SAAO,CAAC,aAAa,QAAQ,CAAC;AAChC;AAIA,MAAM,mBAAgC;AACtC,MAAM,sBAAsB;AAC5B,MAAM,eAAe;AACrB,MAAM,mBAAmB;AAmBlB,MAAM,YAAsC,WAAAC,IAAQ;AAAA,EACjD;AAAA,EACA,UAAyC,oBAAI,IAAI;AAAA,EACzD;AAAA,EAEA,cAAU,gBAAI;AAAA,EAEd,YAAY,MAYT;AACD,UAAM,cAAa,6BAAM,eAAc;AACvC,UAAM,YAAY,GAAG,EAAE,WAAW,KAAK,CAAC;AAExC,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX;AAAA,MACA,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA,eAAe,CAAC;AAAA,MAChB;AAAA,MACA;AAAA,IACF,IAAI,QAAQ,CAAC;AAEb,UAAM,YAAY,eAAW,sCAAuB;AACpD,UAAM,WAAW,UAAU,QAAQ,IAAI,6BAA6B,QAAQ,IAAI;AAChF,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAEA,UAAM,cACJ,aAAa,QAAQ,IAAI,gCAAgC,QAAQ,IAAI;AACvE,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI,MAAM,iEAAiE;AAAA,IACnF;AAGA,QAAI,YAAY;AAChB,QAAI,YAAY;AAChB,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,MAAM,UAAU,YAAY,GAAG;AACrC,UAAI,QAAQ,IAAI;AACd,cAAM,iBAAiB,UAAU,MAAM,MAAM,CAAC;AAC9C,YAAI,aAAa,cAAc,gBAAgB;AAC7C,eAAK,QAAQ;AAAA,YACX;AAAA,YACA,EAAE,OAAO,WAAW,OAAO,UAAU;AAAA,UACvC;AAAA,QACF,OAAO;AACL,sBAAY;AAAA,QACd;AACA,oBAAY,UAAU,MAAM,GAAG,GAAG;AAAA,MACpC;AAAA,IACF;AAEA,UAAM,qBAAqB,WAAW,qBAAqB,QAAQ,IAAI;AAEvE,SAAK,OAAO;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,cAAU,mCAAkB,QAAQ;AAAA,MACpC;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX;AAAA,MACA,UAAU;AAAA,MACV,aAAa,eAAe;AAAA,IAC9B;AAGA,SAAK,OAAO,IAAI,sCAA0B;AAAA,MACxC,WAAW,CAAC,YAAY,KAAK,UAAU,OAAO;AAAA,MAC9C,SAAS,CAAC,OAAO,KAAK,QAAQ,EAAE;AAAA,MAChC,oBAAoB;AAAA,MACpB,oBAAoB;AAAA,MACpB,gBAAgB;AAAA;AAAA,IAClB,CAAC;AAAA,EACH;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,KAAK,SAAS;AAAA,EAC5B;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,gBAAgB,aAAqC;AAC1D,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,WAAW;AACtD,WAAO,IAAI,IAAI,EAAE,OAAO,OAAO,SAAS,OAAU,CAAC;AAAA,EACrD;AAAA,EAEA,cAAc,MAAkF;AAC9F,SAAK,OAAO;AAAA,MACV,GAAG,KAAK;AAAA,MACR,GAAG;AAAA,MACH,UAAU,KAAK,aAAa,aAAY,mCAAkB,KAAK,QAAQ,IAAI,KAAK,KAAK;AAAA,IACvF;AACA,eAAW,UAAU,KAAK,SAAS;AACjC,aAAO,cAAc,KAAK,IAAI;AAAA,IAChC;AAAA,EACF;AAAA,EAEA,WAAW,GAA0B;AACnC,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AAAA,EAEA,OAAO,SAAyE;AAC9E,UAAM,EAAE,cAAc,KAAK,KAAK,eAAe,yCAA4B,IAAI,WAAW,CAAC;AAC3F,UAAM,SAAS,IAAI,iBAAiB,MAAM,EAAE,GAAG,KAAK,KAAK,GAAG,WAAW;AACvE,SAAK,QAAQ,IAAI,MAAM;AACvB,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,UAAU,SAAqC;AA1SvD;AA2SI,QAAI,UAAU,KAAK,KAAK;AACxB,QAAI,QAAQ,WAAW,SAAS,KAAK,QAAQ,WAAW,UAAU,GAAG;AACnE,gBAAU,QAAQ,QAAQ,QAAQ,IAAI;AAAA,IACxC;AAEA,UAAM,QAAQ,UAAM,iCAAkB,KAAK,KAAK,QAAQ,KAAK,KAAK,SAAS;AAC3E,UAAM,MAAM,GAAG,OAAO;AACtB,UAAM,UAAU,EAAE,eAAe,UAAU,KAAK,GAAG;AAEnD,UAAM,SAAS;AAAA,MACb,MAAM;AAAA,MACN,aAAa,OAAO,KAAK,KAAK,UAAU;AAAA,MACxC,UAAU,KAAK,KAAK;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,IACnB;AAEA,QAAI,KAAK,KAAK,MAAO,CAAC,OAAmC,QAAQ,KAAK,KAAK;AAC3E,QAAI,KAAK,KAAK,MAAO,CAAC,OAAmC,QAAQ,KAAK,KAAK;AAC3E,QAAI,KAAK,KAAK,SAAU,CAAC,OAAmC,WAAW,KAAK,KAAK;AAEjF,SAAI,UAAK,KAAK,aAAV,mBAAoB,QAAQ;AAC9B,aAAO,WAAW;AAAA,QAChB,QAAQ,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO;AAAA,UACrC,OAAO,EAAE;AAAA,UACT,OAAO,EAAE;AAAA,UACT,OAAO,EAAE,eAAe,CAAC;AAAA,QAC3B,EAAE;AAAA,MACJ;AAAA,IACF;AAEA,QAAI,KAAK,KAAK,aAAa;AACzB,aAAO,aAAa;AAAA,QAClB,SAAS,KAAK,KAAK,YAAY,YAAY;AAAA,QAC3C,SAAS,KAAK,KAAK,YAAY;AAAA,MACjC;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,IAAI,GAAG,6DAA6D;AACzF,UAAM,SAAS,UAAM,yBAAU,KAAK,SAAS,OAAO;AACpD,WAAO,KAAK,KAAK,UAAU,MAAM,CAAC;AAClC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ,IAAe;AAC3B,UAAM,GAAG,MAAM;AAAA,EACjB;AAAA,EAEA,UAAgB;AACd,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AACZ,eAAW,UAAU,KAAK,SAAS;AACjC,YAAM,OAAO,MAAM;AAAA,IACrB;AACA,SAAK,QAAQ,MAAM;AACnB,UAAM,KAAK,KAAK,MAAM;AAAA,EACxB;AACF;AAEO,MAAM,yBAAmD,WAAAC,iBAAqB;AAAA,EAC3E;AAAA,EACA;AAAA,EAER,cAAU,gBAAI;AAAA,EAEd,YAAY,KAAkB,MAAmC,aAAgC;AAC/F,UAAM,KAAK,WAAW;AACtB,SAAK,OAAO;AACZ,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO;AAAA,EACT;AAAA,EAEA,cAAc,MAAkF;AAC9F,SAAK,OAAO;AAAA,MACV,GAAG,KAAK;AAAA,MACR,GAAG;AAAA,MACH,UAAU,KAAK,aAAa,aAAY,mCAAkB,KAAK,QAAQ,IAAI,KAAK,KAAK;AAAA,IACvF;AAAA,EACF;AAAA,EAEA,MAAgB,MAAqB;AACnC,QAAI,UAAU;AACd,QAAI;AAEJ,UAAM,sBAAsB,IAAI,gBAAAC,MAAc,kBAAkB,EAAE,OAAO;AACzE,UAAM,mBAAe,2CAAoC;AACzD,UAAM,gBAAY,wBAAU,cAAc;AAC1C,UAAM,iBAAiB,IAAI,mBAAM;AAGjC,UAAM,mBAAmB,IAAI,oBAAa;AAE1C,UAAM,kBAAkB,YAAY;AAClC,UAAI,QAAS;AACb,gBAAU;AACV,0BAAoB,MAAM;AAE1B,YAAM,aAAa,MAAM;AAAA,IAC3B;AAEA,UAAM,kBAAkB,OAAO,OAAuB,IAAe,WAAwB;AAE3F,UAAI,OAAO,WAAW,QAAS;AAE/B,YAAM,iBAAiB,MAAM,uCAAqB,WAAW,KAAK;AAClE,UAAI,GAAG,eAAe,oBAAU,MAAM;AACpC,aAAK,QAAQ,KAAK,uDAAuD;AACzE;AAAA,MACF;AACA,SAAG,KAAK,KAAK,UAAU,cAAc,CAAC;AAAA,IACxC;AAEA,UAAM,gBAAgB,CAAC,WAAmB,UAAmB;AAC3D,UAAI,WAAW;AACb,aAAK,MAAM,IAAI,EAAE,WAAW,WAAW,OAAO,WAAW,MAAM,CAAC;AAChE,oBAAY;AAAA,MACd;AAAA,IACF;AAEA,UAAM,kBAAkB,OAAO,WAAwB;AACrD,uBAAiB,QAAQ,KAAK,OAAO;AACnC,YAAI,OAAO,WAAW,QAAS;AAC/B,YAAI,SAAS,iBAAiB,gBAAgB;AAC5C,8BAAoB,MAAM;AAC1B;AAAA,QACF;AACA,4BAAoB,SAAS,IAAI;AAAA,MACnC;AAEA,UAAI,CAAC,SAAS;AACZ,4BAAoB,SAAS;AAAA,MAC/B;AAAA,IACF;AAEA,UAAM,2BAA2B,OAAO,IAAe,WAAwB;AAC7E,uBAAiB,MAAM,qBAAqB;AAC1C,YAAI,OAAO,WAAW,QAAS;AAE/B,cAAM;AAAA,UACJ;AAAA,YACE,MAAM;AAAA,YACN,YAAY,GAAG,QAAQ;AAAA,UACzB;AAAA,UACA;AAAA,UACA;AAAA,QACF;AACA,uBAAe,IAAI;AAAA,MACrB;AAEA,YAAM,gBAAgB,EAAE,MAAM,gBAAgB,GAAG,IAAI,MAAM;AAE3D,qBAAe,IAAI;AAAA,IACrB;AAIA,UAAM,uBAAuB,OAAO,IAAe,WAAwB;AACzE,YAAM,YAAY,CAAC,SAAiB;AAClC,YAAI;AACF,gBAAM,YAAY,KAAK,MAAM,KAAK,SAAS,CAAC;AAC5C,gBAAM,iBAAiB,uCAAqB,MAAM,SAAS;AAE3D,eAAK,aAAa,MAAM,cAAc,EAAE,MAAM,CAAC,UAAU;AACvD,iBAAK,QAAQ;AAAA,cACX,EAAE,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF,CAAC;AAAA,QACH,SAAS,GAAG;AACV,eAAK,QAAQ,MAAM,EAAE,OAAO,EAAE,GAAG,iCAAiC;AAAA,QACpE;AAAA,MACF;AAEA,YAAM,UAAU,CAAC,MAAa;AA5dpC;AA6dQ,aAAK,QAAQ,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB;AAClD,aAAK,gBAAgB;AACrB,YAAI;AAEF,mBAAG,cAAH;AAAA,QACF,QAAQ;AAAA,QAER;AAEA,aAAK,IAAI,KAAK,OAAO,EAAE;AACvB,yBAAiB,OAAO,CAAC;AAAA,MAC3B;AAEA,YAAM,UAAU,MAAM;AAEpB,YAAI,CAAC,SAAS;AACZ,eAAK,QAAQ,MAAM,+BAA+B;AAClD,eAAK,gBAAgB;AAErB,eAAK,IAAI,KAAK,OAAO,EAAE;AACvB,2BAAiB;AAAA,YACf,IAAI,iCAAe;AAAA,cACjB,SAAS;AAAA,cACT,SAAS,EAAE,UAAU;AAAA,YACvB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,UAAU,MAAM;AA1f5B;AA2fQ,aAAK,gBAAgB;AACrB,YAAI;AAGF,mBAAG,cAAH;AAAA,QACF,QAAQ;AAAA,QAER;AACA,aAAK,IAAI,KAAK,OAAO,EAAE;AACvB,uBAAe,IAAI;AACnB,yBAAiB,QAAQ;AAAA,MAC3B;AAGA,SAAG,GAAG,WAAW,SAAS;AAC1B,SAAG,GAAG,SAAS,OAAO;AACtB,SAAG,GAAG,SAAS,OAAO;AACtB,aAAO,iBAAiB,SAAS,OAAO;AAExC,UAAI;AAEF,cAAM,iBAAiB;AAAA,MACzB,UAAE;AAEA,WAAG,IAAI,WAAW,SAAS;AAC3B,WAAG,IAAI,SAAS,OAAO;AACvB,WAAG,IAAI,SAAS,OAAO;AACvB,eAAO,oBAAoB,SAAS,OAAO;AAAA,MAC7C;AAAA,IACF;AAEA,UAAM,iBAAiB,OAAO,WAAwB;AACpD,UAAI,mBAAkC;AAEtC,YAAM,UAAU,IAAI,6BAAgB,KAAK,KAAK,YAAY,YAAY;AACtE,YAAM,oBAAoB,aAAa,OAAO;AAC9C,YAAM,SAAS,kBAAkB,UAAU;AAE3C,UAAI;AACF,cAAM,eAAe,KAAK;AAE1B,eAAO,CAAC,KAAK,UAAU,CAAC,OAAO,SAAS;AACtC,gBAAM,SAAS,MAAM,OAAO,KAAK;AACjC,cAAI,OAAO,QAAS;AACpB,cAAI,OAAO,KAAM;AAEjB,gBAAM,cAAc,OAAO;AAC3B,kBAAQ,YAAY,MAAM;AAAA,YACxB,KAAK;AACH,iCAAmB,YAAY;AAC/B;AAAA,YACF,KAAK;AACH,oBAAM,aAAa,IAAI,UAAU,OAAO,KAAK,YAAY,OAAO,QAAQ,CAAC;AACzE,yBAAW,SAAS,QAAQ,MAAM,WAAW,MAAM,GAAG;AACpD,8BAAc,kBAAmB,KAAK;AACtC,4BAAY;AAAA,cACd;AACA;AAAA,YACF,KAAK;AACH,yBAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,8BAAc,kBAAmB,KAAK;AACtC,4BAAY;AAAA,cACd;AACA,4BAAc,kBAAmB,IAAI;AACrC,mBAAK,MAAM,IAAI,iBAAiB,aAAa;AAC7C,oBAAM,gBAAgB;AACtB,+BAAiB,QAAQ;AACzB;AAAA,YACF,KAAK;AACH,oBAAM,gBAAgB;AACtB,+BAAiB,QAAQ;AACzB;AAAA,YACF,KAAK;AACH,mBAAK,QAAQ;AAAA,gBACX,EAAE,YAAY;AAAA,gBACd;AAAA,cACF;AACA,oBAAM,gBAAgB;AACtB,+BAAiB;AAAA,gBACf,IAAI,2BAAS,+BAA+B,YAAY,OAAO,EAAE;AAAA,cACnE;AACA;AAAA,YACF;AACE,mBAAK,QAAQ,KAAK,yBAAyB,WAAW;AACtD;AAAA,UACJ;AAAA,QACF;AAAA,MACF,UAAE;AACA,eAAO,YAAY;AACnB,YAAI;AACF,gBAAM,kBAAkB,OAAO;AAAA,QACjC,SAAS,GAAG;AACV,eAAK,QAAQ,MAAM,kEAAkE,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF;AAEA,QAAI;AACF,YAAM,KAAK,IAAI,KAAK;AAAA,QAClB,OAAO,OAAkB;AACvB,cAAI;AAGF,kBAAM,gBAAgB,IAAI,gBAAgB;AAC1C,kBAAM,gBAAgB,MAAM,cAAc,MAAM,KAAK,gBAAgB,OAAO,MAAM;AAClF,iBAAK,gBAAgB,OAAO,iBAAiB,SAAS,eAAe,EAAE,MAAM,KAAK,CAAC;AAEnF,kBAAM,QAAQ;AAAA,cACZ,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,gBAAgB,QAAQ;AAAA,gBAChC;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,cACA,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,yBAAyB,IAAI,QAAQ;AAAA,gBAC7C;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,cACA,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,qBAAqB,IAAI,QAAQ;AAAA,gBACzC;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,cACA,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,eAAe,QAAQ;AAAA,gBAC/B;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,YACF;AAEA,gBAAI;AACF,oBAAM,QAAQ,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAAA,YAC9C,UAAE;AAEA,6BAAe,IAAI;AACnB,oBAAM,gBAAgB;AACtB,wBAAM,4BAAc,OAAO,GAAI;AAC/B,mBAAK,gBAAgB,OAAO,oBAAoB,SAAS,aAAa;AAAA,YACxE;AAAA,UACF,SAAS,GAAG;AAEV,gBAAI,aAAa,SAAS,EAAE,SAAS,cAAc;AACjD;AAAA,YACF;AACA,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,QACA;AAAA,UACE,SAAS,KAAK,YAAY;AAAA,QAC5B;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AAEV,UAAI,aAAa,SAAS,EAAE,SAAS,cAAc;AAEjD;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AAEA,YAAM,gBAAgB;AAAA,IACxB;AAAA,EACF;AACF;","names":["import_utils","BaseTTS","BaseSynthesizeStream","tokenizeBasic"]}
1
+ {"version":3,"sources":["../../src/inference/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { WebSocket } from 'ws';\nimport { APIError, APIStatusError, APITimeoutError } from '../_exceptions.js';\nimport { AudioByteStream } from '../audio.js';\nimport { ConnectionPool } from '../connection_pool.js';\nimport { type LanguageCode, normalizeLanguage } from '../language.js';\nimport { log } from '../log.js';\nimport { createStreamChannel } from '../stream/stream_channel.js';\nimport { basic as tokenizeBasic } from '../tokenize/index.js';\nimport type { ChunkedStream } from '../tts/index.js';\nimport { SynthesizeStream as BaseSynthesizeStream, TTS as BaseTTS } from '../tts/index.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport {\n Event,\n Future,\n Task,\n cancelAndWait,\n combineSignals,\n shortuuid,\n waitUntilTimeout,\n} from '../utils.js';\nimport {\n type TtsClientEvent,\n type TtsServerEvent,\n ttsClientEventSchema,\n ttsServerEventSchema,\n} from './api_protos.js';\nimport { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js';\n\nexport type CartesiaModels =\n | 'cartesia/sonic-3'\n | 'cartesia/sonic-2'\n | 'cartesia/sonic-turbo'\n | 'cartesia/sonic';\n\nexport type DeepgramTTSModels = 'deepgram/aura' | 'deepgram/aura-2';\n\nexport type ElevenlabsModels =\n | 'elevenlabs/eleven_flash_v2'\n | 'elevenlabs/eleven_flash_v2_5'\n | 'elevenlabs/eleven_turbo_v2'\n | 'elevenlabs/eleven_turbo_v2_5'\n | 'elevenlabs/eleven_multilingual_v2';\n\nexport type InworldModels =\n | 'inworld/inworld-tts-1.5-max'\n | 'inworld/inworld-tts-1.5-mini'\n | 'inworld/inworld-tts-1-max'\n | 'inworld/inworld-tts-1';\n\nexport type RimeModels = 'rime/arcana' | 'rime/mistv2';\n\nexport interface CartesiaOptions {\n emotion?: string;\n /** Maximum duration of audio in seconds. */\n duration?: number;\n /** Speech speed. Default: not specified. */\n speed?: 'slow' | 'normal' | 'fast' | number;\n volume?: number;\n max_buffer_delay_ms?: number;\n add_timestamps?: boolean;\n add_phoneme_timestamps?: boolean;\n use_normalized_timestamps?: boolean;\n}\n\nexport interface ElevenlabsOptions {\n /** Inactivity timeout in seconds. Default: 60. */\n inactivity_timeout?: number;\n /** Text normalization mode. Default: \"auto\". */\n apply_text_normalization?: 'auto' | 'off' | 'on';\n auto_mode?: boolean;\n enable_logging?: boolean;\n enable_ssml_parsing?: boolean;\n sync_alignment?: boolean;\n language_code?: string;\n /** Voice stability tuning, typically in the range [0, 1]. */\n stability?: number;\n /** Voice similarity tuning, typically in the range [0, 1]. */\n similarity_boost?: number;\n /** Style exaggeration tuning, typically in the range [0, 1]. */\n style?: number;\n /** Speech speed multiplier. */\n speed?: number;\n use_speaker_boost?: boolean;\n chunk_length_schedule?: number[];\n preferred_alignment?: string;\n}\n\nexport interface DeepgramTTSOptions {\n /** Default: false. */\n mip_opt_out?: boolean;\n}\n\nexport interface RimeOptions {\n /** Default 1.0, <1 = faster, >1 = slower. */\n speed_alpha?: number;\n /** Default false. */\n pause_between_brackets?: boolean;\n /** Default false. */\n phonemize_between_brackets?: boolean;\n /** Comma-separated speed factors for [bracketed] words. */\n inline_speed_alpha?: string;\n /** Default false. */\n no_text_normalization?: boolean;\n}\n\nexport interface InworldOptions {\n /** Range >0.5, <=1.5. */\n speaking_rate?: number;\n /** Range 0-2. */\n temperature?: number;\n timestamp_type?: 'TIMESTAMP_TYPE_UNSPECIFIED' | 'WORD' | 'CHARACTER';\n apply_text_normalization?: 'APPLY_TEXT_NORMALIZATION_UNSPECIFIED' | 'ON' | 'OFF';\n /** @deprecated Backward-compatible alias. Use `apply_text_normalization`. */\n text_normalization?: 'ON' | 'OFF';\n}\n\ntype _TTSModels =\n | CartesiaModels\n | DeepgramTTSModels\n | ElevenlabsModels\n | RimeModels\n | InworldModels;\n\nexport type TTSModels =\n | CartesiaModels\n | DeepgramTTSModels\n | ElevenlabsModels\n | RimeModels\n | InworldModels\n | AnyString;\n\nexport type ModelWithVoice = `${_TTSModels}:${string}` | TTSModels;\n\nexport type TTSOptions<TModel extends TTSModels> = TModel extends CartesiaModels\n ? CartesiaOptions\n : TModel extends DeepgramTTSModels\n ? DeepgramTTSOptions\n : TModel extends ElevenlabsModels\n ? ElevenlabsOptions\n : TModel extends RimeModels\n ? RimeOptions\n : TModel extends InworldModels\n ? InworldOptions\n : Record<string, unknown>;\n\n/** Parse a model string into [model, voice]. Voice is undefined if not specified. */\nexport function parseTTSModelString(model: string): [string, string | undefined] {\n const idx = model.lastIndexOf(':');\n if (idx !== -1) {\n return [model.slice(0, idx), model.slice(idx + 1)];\n }\n return [model, undefined];\n}\n\n/** A fallback model with optional extra configuration. Extra fields are passed through to the provider. */\nexport interface TTSFallbackModel {\n /** Model name (e.g. \"cartesia/sonic\", \"elevenlabs/eleven_flash_v2\", \"rime/arcana\"). */\n model: string;\n /** Voice to use for the model. */\n voice: string;\n /** Extra configuration for the model. */\n extraKwargs?: Record<string, unknown>;\n}\n\nexport type TTSFallbackModelType = TTSFallbackModel | string;\n\n/** Normalize a single or list of FallbackModelType into TTSFallbackModel[]. */\nexport function normalizeTTSFallback(\n fallback: TTSFallbackModelType | TTSFallbackModelType[],\n): TTSFallbackModel[] {\n const makeFallback = (model: TTSFallbackModelType): TTSFallbackModel => {\n if (typeof model === 'string') {\n const [name, voice] = parseTTSModelString(model);\n return { model: name, voice: voice ?? '' };\n }\n return model;\n };\n\n if (Array.isArray(fallback)) {\n return fallback.map(makeFallback);\n }\n return [makeFallback(fallback)];\n}\n\ntype TTSEncoding = 'pcm_s16le';\n\nconst DEFAULT_ENCODING: TTSEncoding = 'pcm_s16le';\nconst DEFAULT_SAMPLE_RATE = 16000;\nconst NUM_CHANNELS = 1;\nconst DEFAULT_LANGUAGE = 'en';\n\nexport interface InferenceTTSOptions<TModel extends TTSModels> {\n model?: TModel;\n voice?: string;\n language?: LanguageCode;\n encoding: TTSEncoding;\n sampleRate: number;\n baseURL: string;\n apiKey: string;\n apiSecret: string;\n /** Flat provider-specific inference options forwarded as the `extra` payload field. */\n modelOptions: TTSOptions<TModel>;\n fallback?: TTSFallbackModel[];\n connOptions?: APIConnectOptions;\n}\n\n/**\n * Livekit Cloud Inference TTS\n */\nexport class TTS<TModel extends TTSModels> extends BaseTTS {\n private opts: InferenceTTSOptions<TModel>;\n private streams: Set<SynthesizeStream<TModel>> = new Set();\n pool: ConnectionPool<WebSocket>;\n\n #logger = log();\n\n constructor(opts: {\n model: TModel;\n voice?: string;\n language?: string;\n baseURL?: string;\n encoding?: TTSEncoding;\n sampleRate?: number;\n apiKey?: string;\n apiSecret?: string;\n /** Flat provider-specific inference options forwarded as the `extra` payload field. */\n modelOptions?: TTSOptions<TModel>;\n fallback?: TTSFallbackModelType | TTSFallbackModelType[];\n connOptions?: APIConnectOptions;\n }) {\n const sampleRate = opts?.sampleRate ?? DEFAULT_SAMPLE_RATE;\n super(sampleRate, 1, { streaming: true });\n\n const {\n model,\n voice,\n language = DEFAULT_LANGUAGE,\n baseURL,\n encoding = DEFAULT_ENCODING,\n apiKey,\n apiSecret,\n modelOptions = {} as TTSOptions<TModel>,\n fallback,\n connOptions,\n } = opts || {};\n\n const lkBaseURL = baseURL || getDefaultInferenceUrl();\n const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;\n if (!lkApiKey) {\n throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');\n }\n\n const lkApiSecret =\n apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;\n if (!lkApiSecret) {\n throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');\n }\n\n // read voice id from the model if provided: \"provider/model:voice_id\"\n let nextModel = model;\n let nextVoice = voice;\n if (typeof nextModel === 'string') {\n const idx = nextModel.lastIndexOf(':');\n if (idx !== -1) {\n const voiceFromModel = nextModel.slice(idx + 1);\n if (nextVoice && nextVoice !== voiceFromModel) {\n this.#logger.warn(\n '`voice` is provided via both argument and model, using the one from the argument',\n { voice: nextVoice, model: nextModel },\n );\n } else {\n nextVoice = voiceFromModel;\n }\n nextModel = nextModel.slice(0, idx) as TModel;\n }\n }\n\n const normalizedFallback = fallback ? normalizeTTSFallback(fallback) : undefined;\n\n this.opts = {\n model: nextModel,\n voice: nextVoice,\n language: normalizeLanguage(language),\n encoding,\n sampleRate,\n baseURL: lkBaseURL,\n apiKey: lkApiKey,\n apiSecret: lkApiSecret,\n modelOptions,\n fallback: normalizedFallback,\n connOptions: connOptions ?? DEFAULT_API_CONNECT_OPTIONS,\n };\n\n // Initialize connection pool\n this.pool = new ConnectionPool<WebSocket>({\n connectCb: (timeout) => this.connectWs(timeout),\n closeCb: (ws) => this.closeWs(ws),\n maxSessionDuration: 300_000,\n markRefreshedOnGet: true,\n connectTimeout: 10_000, // 10 seconds default\n });\n }\n\n get label() {\n return 'inference.TTS';\n }\n\n get model(): string {\n return this.opts.model ?? 'unknown';\n }\n\n get provider(): string {\n return 'livekit';\n }\n\n static fromModelString(modelString: string): TTS<AnyString> {\n const [model, voice] = parseTTSModelString(modelString);\n return new TTS({ model, voice: voice || undefined });\n }\n\n updateOptions(opts: Partial<Pick<InferenceTTSOptions<TModel>, 'model' | 'voice' | 'language'>>) {\n this.opts = {\n ...this.opts,\n ...opts,\n language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,\n };\n for (const stream of this.streams) {\n stream.updateOptions(this.opts);\n }\n }\n\n synthesize(_: string): ChunkedStream {\n throw new Error('ChunkedStream is not implemented');\n }\n\n stream(options?: { connOptions?: APIConnectOptions }): SynthesizeStream<TModel> {\n const { connOptions = this.opts.connOptions ?? DEFAULT_API_CONNECT_OPTIONS } = options || {};\n const stream = new SynthesizeStream(this, { ...this.opts }, connOptions);\n this.streams.add(stream);\n return stream;\n }\n\n async connectWs(timeout: number): Promise<WebSocket> {\n let baseURL = this.opts.baseURL;\n if (baseURL.startsWith('http://') || baseURL.startsWith('https://')) {\n baseURL = baseURL.replace('http', 'ws');\n }\n\n const token = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);\n const url = `${baseURL}/tts`;\n const headers = { Authorization: `Bearer ${token}` } as Record<string, string>;\n\n const params = {\n type: 'session.create',\n sample_rate: String(this.opts.sampleRate),\n encoding: this.opts.encoding,\n extra: this.opts.modelOptions,\n } as Record<string, unknown>;\n\n if (this.opts.voice) (params as Record<string, unknown>).voice = this.opts.voice;\n if (this.opts.model) (params as Record<string, unknown>).model = this.opts.model;\n if (this.opts.language) (params as Record<string, unknown>).language = this.opts.language;\n\n if (this.opts.fallback?.length) {\n params.fallback = {\n models: this.opts.fallback.map((m) => ({\n model: m.model,\n voice: m.voice,\n extra: m.extraKwargs ?? {},\n })),\n };\n }\n\n if (this.opts.connOptions) {\n params.connection = {\n timeout: this.opts.connOptions.timeoutMs / 1000,\n retries: this.opts.connOptions.maxRetry,\n };\n }\n\n this.#logger.debug({ url }, 'inference.TTS creating new websocket connection (pool miss)');\n const socket = await connectWs(url, headers, timeout);\n socket.send(JSON.stringify(params));\n return socket;\n }\n\n async closeWs(ws: WebSocket) {\n await ws.close();\n }\n\n prewarm(): void {\n this.pool.prewarm();\n }\n\n async close() {\n for (const stream of this.streams) {\n await stream.close();\n }\n this.streams.clear();\n await this.pool.close();\n }\n}\n\nexport class SynthesizeStream<TModel extends TTSModels> extends BaseSynthesizeStream {\n private opts: InferenceTTSOptions<TModel>;\n private tts: TTS<TModel>;\n\n #logger = log();\n\n constructor(tts: TTS<TModel>, opts: InferenceTTSOptions<TModel>, connOptions: APIConnectOptions) {\n super(tts, connOptions);\n this.opts = opts;\n this.tts = tts;\n }\n\n get label() {\n return 'inference.SynthesizeStream';\n }\n\n updateOptions(opts: Partial<Pick<InferenceTTSOptions<TModel>, 'model' | 'voice' | 'language'>>) {\n this.opts = {\n ...this.opts,\n ...opts,\n language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,\n };\n }\n\n protected async run(): Promise<void> {\n let closing = false;\n let lastFrame: AudioFrame | undefined;\n\n const sendTokenizerStream = new tokenizeBasic.SentenceTokenizer().stream();\n const eventChannel = createStreamChannel<TtsServerEvent>();\n const requestId = shortuuid('tts_request_');\n const inputSentEvent = new Event();\n\n // Signal for protocol-driven completion (when 'done' message is received)\n const completionFuture = new Future<void>();\n\n const resourceCleanup = async () => {\n if (closing) return;\n closing = true;\n sendTokenizerStream.close();\n // close() returns a promise; don't leak it\n await eventChannel.close();\n };\n\n const sendClientEvent = async (event: TtsClientEvent, ws: WebSocket, signal: AbortSignal) => {\n // Don't send events to a closed WebSocket or aborted controller\n if (signal.aborted || closing) return;\n\n const validatedEvent = await ttsClientEventSchema.parseAsync(event);\n if (ws.readyState !== WebSocket.OPEN) {\n this.#logger.warn('Trying to send client TTS event to a closed WebSocket');\n return;\n }\n ws.send(JSON.stringify(validatedEvent));\n };\n\n const sendLastFrame = (segmentId: string, final: boolean) => {\n if (lastFrame) {\n this.queue.put({ requestId, segmentId, frame: lastFrame, final });\n lastFrame = undefined;\n }\n };\n\n const createInputTask = async (signal: AbortSignal) => {\n for await (const data of this.input) {\n if (signal.aborted || closing) break;\n if (data === SynthesizeStream.FLUSH_SENTINEL) {\n sendTokenizerStream.flush();\n continue;\n }\n sendTokenizerStream.pushText(data);\n }\n // Only call endInput if the stream hasn't been closed by cleanup\n if (!closing) {\n sendTokenizerStream.endInput();\n }\n };\n\n const createSentenceStreamTask = async (ws: WebSocket, signal: AbortSignal) => {\n for await (const ev of sendTokenizerStream) {\n if (signal.aborted || closing) break;\n\n await sendClientEvent(\n {\n type: 'input_transcript',\n transcript: ev.token + ' ',\n },\n ws,\n signal,\n );\n inputSentEvent.set();\n }\n\n await sendClientEvent({ type: 'session.flush' }, ws, signal);\n // needed in case empty input is sent\n inputSentEvent.set();\n };\n\n // Handles WebSocket message routing and error handling\n // Completes based on protocol messages, NOT on ws.close()\n const createWsListenerTask = async (ws: WebSocket, signal: AbortSignal) => {\n const onMessage = (data: Buffer) => {\n try {\n const eventJson = JSON.parse(data.toString()) as Record<string, unknown>;\n const validatedEvent = ttsServerEventSchema.parse(eventJson);\n // writer.write returns a promise; avoid unhandled rejections if stream is closed\n void eventChannel.write(validatedEvent).catch((error) => {\n this.#logger.debug(\n { error },\n 'Failed writing TTS event to stream channel (likely closed)',\n );\n });\n } catch (e) {\n this.#logger.error({ error: e }, 'Error parsing WebSocket message');\n }\n };\n\n const onError = (e: Error) => {\n this.#logger.error({ error: e }, 'WebSocket error');\n void resourceCleanup();\n try {\n // If the ws is misbehaving, hard-stop it immediately to avoid buffering.\n ws.terminate?.();\n } catch {\n // ignore\n }\n // Ensure this ws is not reused\n this.tts.pool.remove(ws);\n completionFuture.reject(e);\n };\n\n const onClose = () => {\n // WebSocket closed unexpectedly (not by us)\n if (!closing) {\n this.#logger.error('WebSocket closed unexpectedly');\n void resourceCleanup();\n // Ensure this ws is not reused\n this.tts.pool.remove(ws);\n completionFuture.reject(\n new APIStatusError({\n message: 'Gateway connection closed unexpectedly',\n options: { requestId },\n }),\n );\n }\n };\n\n const onAbort = () => {\n void resourceCleanup();\n try {\n // On interruption/abort, close the websocket immediately so the server stops streaming\n // and the ws library doesn't buffer unread frames in memory.\n ws.terminate?.();\n } catch {\n // ignore\n }\n this.tts.pool.remove(ws);\n inputSentEvent.set();\n completionFuture.resolve();\n };\n\n // Attach listeners\n ws.on('message', onMessage);\n ws.on('error', onError);\n ws.on('close', onClose);\n signal.addEventListener('abort', onAbort);\n\n try {\n // Wait for protocol-driven completion or error\n await completionFuture.await;\n } finally {\n // IMPORTANT: Remove listeners so connection can be reused\n ws.off('message', onMessage);\n ws.off('error', onError);\n ws.off('close', onClose);\n signal.removeEventListener('abort', onAbort);\n }\n };\n\n const createRecvTask = async (signal: AbortSignal) => {\n let currentSessionId: string | null = null;\n const recvTimeoutMs = this.connOptions.timeoutMs;\n\n const bstream = new AudioByteStream(this.opts.sampleRate, NUM_CHANNELS);\n const serverEventStream = eventChannel.stream();\n const reader = serverEventStream.getReader();\n\n try {\n await inputSentEvent.wait();\n\n while (!this.closed && !signal.aborted) {\n const result = await waitUntilTimeout(\n reader.read(),\n recvTimeoutMs,\n () => new APITimeoutError({ message: 'TTS recv idle timeout' }),\n );\n\n if (signal.aborted) return;\n if (result.done) return;\n\n const serverEvent = result.value;\n switch (serverEvent.type) {\n case 'session.created':\n currentSessionId = serverEvent.session_id;\n break;\n case 'output_audio':\n const base64Data = new Int8Array(Buffer.from(serverEvent.audio, 'base64'));\n for (const frame of bstream.write(base64Data.buffer)) {\n sendLastFrame(currentSessionId!, false);\n lastFrame = frame;\n }\n break;\n case 'done':\n for (const frame of bstream.flush()) {\n sendLastFrame(currentSessionId!, false);\n lastFrame = frame;\n }\n sendLastFrame(currentSessionId!, true);\n this.queue.put(SynthesizeStream.END_OF_STREAM);\n await resourceCleanup();\n completionFuture.resolve();\n return;\n case 'session.closed':\n await resourceCleanup();\n completionFuture.resolve();\n return;\n case 'error':\n this.#logger.error(\n { serverEvent },\n 'Received error message from LiveKit TTS WebSocket',\n );\n await resourceCleanup();\n completionFuture.reject(\n new APIError(`LiveKit TTS returned error: ${serverEvent.message}`),\n );\n return;\n default:\n this.#logger.warn('Unexpected message %s', serverEvent);\n break;\n }\n }\n } catch (e) {\n if (e instanceof APITimeoutError) {\n this.#logger.warn('TTS recv task timed out waiting for server message');\n await resourceCleanup();\n completionFuture.reject(e);\n return;\n }\n throw e;\n } finally {\n reader.releaseLock();\n try {\n await serverEventStream.cancel();\n } catch (e) {\n this.#logger.debug('Error cancelling serverEventStream (may already be cancelled):', e);\n }\n }\n };\n\n try {\n await this.tts.pool.withConnection(\n async (ws: WebSocket) => {\n try {\n // IMPORTANT: don't cancel the stream's controller on normal completion,\n // otherwise the pool will remove+close the ws and every run becomes a pool miss.\n const runController = new AbortController();\n const onStreamAbort = () => runController.abort(this.abortController.signal.reason);\n this.abortController.signal.addEventListener('abort', onStreamAbort, { once: true });\n\n const tasks = [\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createInputTask(combined);\n },\n undefined,\n 'inference-tts-input',\n ),\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createSentenceStreamTask(ws, combined);\n },\n undefined,\n 'inference-tts-sentence',\n ),\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createWsListenerTask(ws, combined);\n },\n undefined,\n 'inference-tts-ws-listener',\n ),\n Task.from(\n async (controller) => {\n const combined = combineSignals(runController.signal, controller.signal);\n await createRecvTask(combined);\n },\n undefined,\n 'inference-tts-recv',\n ),\n ];\n\n try {\n await Promise.all(tasks.map((t) => t.result));\n } finally {\n // Mirror python finally: unblock recv and cancel all tasks.\n inputSentEvent.set();\n await resourceCleanup();\n await cancelAndWait(tasks, 5000);\n this.abortController.signal.removeEventListener('abort', onStreamAbort);\n }\n } catch (e) {\n // If aborted, don't throw - let cleanup handle it\n if (e instanceof Error && e.name === 'AbortError') {\n return;\n }\n throw e;\n }\n },\n {\n timeout: this.connOptions.timeoutMs,\n },\n );\n } catch (e) {\n // Handle connection errors\n if (e instanceof Error && e.name === 'AbortError') {\n // Abort is expected during normal shutdown\n return;\n }\n throw e;\n } finally {\n // Ensure cleanup always runs (and don't leak the promise)\n await resourceCleanup();\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,gBAA0B;AAC1B,wBAA0D;AAC1D,mBAAgC;AAChC,6BAA+B;AAC/B,sBAAqD;AACrD,iBAAoB;AACpB,4BAAoC;AACpC,sBAAuC;AAEvC,iBAAyE;AACzE,mBAAoE;AACpE,mBAQO;AACP,wBAKO;AACP,IAAAA,gBAAqF;AAwH9E,SAAS,oBAAoB,OAA6C;AAC/E,QAAM,MAAM,MAAM,YAAY,GAAG;AACjC,MAAI,QAAQ,IAAI;AACd,WAAO,CAAC,MAAM,MAAM,GAAG,GAAG,GAAG,MAAM,MAAM,MAAM,CAAC,CAAC;AAAA,EACnD;AACA,SAAO,CAAC,OAAO,MAAS;AAC1B;AAeO,SAAS,qBACd,UACoB;AACpB,QAAM,eAAe,CAAC,UAAkD;AACtE,QAAI,OAAO,UAAU,UAAU;AAC7B,YAAM,CAAC,MAAM,KAAK,IAAI,oBAAoB,KAAK;AAC/C,aAAO,EAAE,OAAO,MAAM,OAAO,SAAS,GAAG;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,QAAQ,GAAG;AAC3B,WAAO,SAAS,IAAI,YAAY;AAAA,EAClC;AACA,SAAO,CAAC,aAAa,QAAQ,CAAC;AAChC;AAIA,MAAM,mBAAgC;AACtC,MAAM,sBAAsB;AAC5B,MAAM,eAAe;AACrB,MAAM,mBAAmB;AAoBlB,MAAM,YAAsC,WAAAC,IAAQ;AAAA,EACjD;AAAA,EACA,UAAyC,oBAAI,IAAI;AAAA,EACzD;AAAA,EAEA,cAAU,gBAAI;AAAA,EAEd,YAAY,MAaT;AACD,UAAM,cAAa,6BAAM,eAAc;AACvC,UAAM,YAAY,GAAG,EAAE,WAAW,KAAK,CAAC;AAExC,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX;AAAA,MACA,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA,eAAe,CAAC;AAAA,MAChB;AAAA,MACA;AAAA,IACF,IAAI,QAAQ,CAAC;AAEb,UAAM,YAAY,eAAW,sCAAuB;AACpD,UAAM,WAAW,UAAU,QAAQ,IAAI,6BAA6B,QAAQ,IAAI;AAChF,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAEA,UAAM,cACJ,aAAa,QAAQ,IAAI,gCAAgC,QAAQ,IAAI;AACvE,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI,MAAM,iEAAiE;AAAA,IACnF;AAGA,QAAI,YAAY;AAChB,QAAI,YAAY;AAChB,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,MAAM,UAAU,YAAY,GAAG;AACrC,UAAI,QAAQ,IAAI;AACd,cAAM,iBAAiB,UAAU,MAAM,MAAM,CAAC;AAC9C,YAAI,aAAa,cAAc,gBAAgB;AAC7C,eAAK,QAAQ;AAAA,YACX;AAAA,YACA,EAAE,OAAO,WAAW,OAAO,UAAU;AAAA,UACvC;AAAA,QACF,OAAO;AACL,sBAAY;AAAA,QACd;AACA,oBAAY,UAAU,MAAM,GAAG,GAAG;AAAA,MACpC;AAAA,IACF;AAEA,UAAM,qBAAqB,WAAW,qBAAqB,QAAQ,IAAI;AAEvE,SAAK,OAAO;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,cAAU,mCAAkB,QAAQ;AAAA,MACpC;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX;AAAA,MACA,UAAU;AAAA,MACV,aAAa,eAAe;AAAA,IAC9B;AAGA,SAAK,OAAO,IAAI,sCAA0B;AAAA,MACxC,WAAW,CAAC,YAAY,KAAK,UAAU,OAAO;AAAA,MAC9C,SAAS,CAAC,OAAO,KAAK,QAAQ,EAAE;AAAA,MAChC,oBAAoB;AAAA,MACpB,oBAAoB;AAAA,MACpB,gBAAgB;AAAA;AAAA,IAClB,CAAC;AAAA,EACH;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,KAAK,SAAS;AAAA,EAC5B;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,gBAAgB,aAAqC;AAC1D,UAAM,CAAC,OAAO,KAAK,IAAI,oBAAoB,WAAW;AACtD,WAAO,IAAI,IAAI,EAAE,OAAO,OAAO,SAAS,OAAU,CAAC;AAAA,EACrD;AAAA,EAEA,cAAc,MAAkF;AAC9F,SAAK,OAAO;AAAA,MACV,GAAG,KAAK;AAAA,MACR,GAAG;AAAA,MACH,UAAU,KAAK,aAAa,aAAY,mCAAkB,KAAK,QAAQ,IAAI,KAAK,KAAK;AAAA,IACvF;AACA,eAAW,UAAU,KAAK,SAAS;AACjC,aAAO,cAAc,KAAK,IAAI;AAAA,IAChC;AAAA,EACF;AAAA,EAEA,WAAW,GAA0B;AACnC,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AAAA,EAEA,OAAO,SAAyE;AAC9E,UAAM,EAAE,cAAc,KAAK,KAAK,eAAe,yCAA4B,IAAI,WAAW,CAAC;AAC3F,UAAM,SAAS,IAAI,iBAAiB,MAAM,EAAE,GAAG,KAAK,KAAK,GAAG,WAAW;AACvE,SAAK,QAAQ,IAAI,MAAM;AACvB,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,UAAU,SAAqC;AA1VvD;AA2VI,QAAI,UAAU,KAAK,KAAK;AACxB,QAAI,QAAQ,WAAW,SAAS,KAAK,QAAQ,WAAW,UAAU,GAAG;AACnE,gBAAU,QAAQ,QAAQ,QAAQ,IAAI;AAAA,IACxC;AAEA,UAAM,QAAQ,UAAM,iCAAkB,KAAK,KAAK,QAAQ,KAAK,KAAK,SAAS;AAC3E,UAAM,MAAM,GAAG,OAAO;AACtB,UAAM,UAAU,EAAE,eAAe,UAAU,KAAK,GAAG;AAEnD,UAAM,SAAS;AAAA,MACb,MAAM;AAAA,MACN,aAAa,OAAO,KAAK,KAAK,UAAU;AAAA,MACxC,UAAU,KAAK,KAAK;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,IACnB;AAEA,QAAI,KAAK,KAAK,MAAO,CAAC,OAAmC,QAAQ,KAAK,KAAK;AAC3E,QAAI,KAAK,KAAK,MAAO,CAAC,OAAmC,QAAQ,KAAK,KAAK;AAC3E,QAAI,KAAK,KAAK,SAAU,CAAC,OAAmC,WAAW,KAAK,KAAK;AAEjF,SAAI,UAAK,KAAK,aAAV,mBAAoB,QAAQ;AAC9B,aAAO,WAAW;AAAA,QAChB,QAAQ,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO;AAAA,UACrC,OAAO,EAAE;AAAA,UACT,OAAO,EAAE;AAAA,UACT,OAAO,EAAE,eAAe,CAAC;AAAA,QAC3B,EAAE;AAAA,MACJ;AAAA,IACF;AAEA,QAAI,KAAK,KAAK,aAAa;AACzB,aAAO,aAAa;AAAA,QAClB,SAAS,KAAK,KAAK,YAAY,YAAY;AAAA,QAC3C,SAAS,KAAK,KAAK,YAAY;AAAA,MACjC;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,IAAI,GAAG,6DAA6D;AACzF,UAAM,SAAS,UAAM,yBAAU,KAAK,SAAS,OAAO;AACpD,WAAO,KAAK,KAAK,UAAU,MAAM,CAAC;AAClC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ,IAAe;AAC3B,UAAM,GAAG,MAAM;AAAA,EACjB;AAAA,EAEA,UAAgB;AACd,SAAK,KAAK,QAAQ;AAAA,EACpB;AAAA,EAEA,MAAM,QAAQ;AACZ,eAAW,UAAU,KAAK,SAAS;AACjC,YAAM,OAAO,MAAM;AAAA,IACrB;AACA,SAAK,QAAQ,MAAM;AACnB,UAAM,KAAK,KAAK,MAAM;AAAA,EACxB;AACF;AAEO,MAAM,yBAAmD,WAAAC,iBAAqB;AAAA,EAC3E;AAAA,EACA;AAAA,EAER,cAAU,gBAAI;AAAA,EAEd,YAAY,KAAkB,MAAmC,aAAgC;AAC/F,UAAM,KAAK,WAAW;AACtB,SAAK,OAAO;AACZ,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO;AAAA,EACT;AAAA,EAEA,cAAc,MAAkF;AAC9F,SAAK,OAAO;AAAA,MACV,GAAG,KAAK;AAAA,MACR,GAAG;AAAA,MACH,UAAU,KAAK,aAAa,aAAY,mCAAkB,KAAK,QAAQ,IAAI,KAAK,KAAK;AAAA,IACvF;AAAA,EACF;AAAA,EAEA,MAAgB,MAAqB;AACnC,QAAI,UAAU;AACd,QAAI;AAEJ,UAAM,sBAAsB,IAAI,gBAAAC,MAAc,kBAAkB,EAAE,OAAO;AACzE,UAAM,mBAAe,2CAAoC;AACzD,UAAM,gBAAY,wBAAU,cAAc;AAC1C,UAAM,iBAAiB,IAAI,mBAAM;AAGjC,UAAM,mBAAmB,IAAI,oBAAa;AAE1C,UAAM,kBAAkB,YAAY;AAClC,UAAI,QAAS;AACb,gBAAU;AACV,0BAAoB,MAAM;AAE1B,YAAM,aAAa,MAAM;AAAA,IAC3B;AAEA,UAAM,kBAAkB,OAAO,OAAuB,IAAe,WAAwB;AAE3F,UAAI,OAAO,WAAW,QAAS;AAE/B,YAAM,iBAAiB,MAAM,uCAAqB,WAAW,KAAK;AAClE,UAAI,GAAG,eAAe,oBAAU,MAAM;AACpC,aAAK,QAAQ,KAAK,uDAAuD;AACzE;AAAA,MACF;AACA,SAAG,KAAK,KAAK,UAAU,cAAc,CAAC;AAAA,IACxC;AAEA,UAAM,gBAAgB,CAAC,WAAmB,UAAmB;AAC3D,UAAI,WAAW;AACb,aAAK,MAAM,IAAI,EAAE,WAAW,WAAW,OAAO,WAAW,MAAM,CAAC;AAChE,oBAAY;AAAA,MACd;AAAA,IACF;AAEA,UAAM,kBAAkB,OAAO,WAAwB;AACrD,uBAAiB,QAAQ,KAAK,OAAO;AACnC,YAAI,OAAO,WAAW,QAAS;AAC/B,YAAI,SAAS,iBAAiB,gBAAgB;AAC5C,8BAAoB,MAAM;AAC1B;AAAA,QACF;AACA,4BAAoB,SAAS,IAAI;AAAA,MACnC;AAEA,UAAI,CAAC,SAAS;AACZ,4BAAoB,SAAS;AAAA,MAC/B;AAAA,IACF;AAEA,UAAM,2BAA2B,OAAO,IAAe,WAAwB;AAC7E,uBAAiB,MAAM,qBAAqB;AAC1C,YAAI,OAAO,WAAW,QAAS;AAE/B,cAAM;AAAA,UACJ;AAAA,YACE,MAAM;AAAA,YACN,YAAY,GAAG,QAAQ;AAAA,UACzB;AAAA,UACA;AAAA,UACA;AAAA,QACF;AACA,uBAAe,IAAI;AAAA,MACrB;AAEA,YAAM,gBAAgB,EAAE,MAAM,gBAAgB,GAAG,IAAI,MAAM;AAE3D,qBAAe,IAAI;AAAA,IACrB;AAIA,UAAM,uBAAuB,OAAO,IAAe,WAAwB;AACzE,YAAM,YAAY,CAAC,SAAiB;AAClC,YAAI;AACF,gBAAM,YAAY,KAAK,MAAM,KAAK,SAAS,CAAC;AAC5C,gBAAM,iBAAiB,uCAAqB,MAAM,SAAS;AAE3D,eAAK,aAAa,MAAM,cAAc,EAAE,MAAM,CAAC,UAAU;AACvD,iBAAK,QAAQ;AAAA,cACX,EAAE,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF,CAAC;AAAA,QACH,SAAS,GAAG;AACV,eAAK,QAAQ,MAAM,EAAE,OAAO,EAAE,GAAG,iCAAiC;AAAA,QACpE;AAAA,MACF;AAEA,YAAM,UAAU,CAAC,MAAa;AA5gBpC;AA6gBQ,aAAK,QAAQ,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB;AAClD,aAAK,gBAAgB;AACrB,YAAI;AAEF,mBAAG,cAAH;AAAA,QACF,QAAQ;AAAA,QAER;AAEA,aAAK,IAAI,KAAK,OAAO,EAAE;AACvB,yBAAiB,OAAO,CAAC;AAAA,MAC3B;AAEA,YAAM,UAAU,MAAM;AAEpB,YAAI,CAAC,SAAS;AACZ,eAAK,QAAQ,MAAM,+BAA+B;AAClD,eAAK,gBAAgB;AAErB,eAAK,IAAI,KAAK,OAAO,EAAE;AACvB,2BAAiB;AAAA,YACf,IAAI,iCAAe;AAAA,cACjB,SAAS;AAAA,cACT,SAAS,EAAE,UAAU;AAAA,YACvB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,YAAM,UAAU,MAAM;AA1iB5B;AA2iBQ,aAAK,gBAAgB;AACrB,YAAI;AAGF,mBAAG,cAAH;AAAA,QACF,QAAQ;AAAA,QAER;AACA,aAAK,IAAI,KAAK,OAAO,EAAE;AACvB,uBAAe,IAAI;AACnB,yBAAiB,QAAQ;AAAA,MAC3B;AAGA,SAAG,GAAG,WAAW,SAAS;AAC1B,SAAG,GAAG,SAAS,OAAO;AACtB,SAAG,GAAG,SAAS,OAAO;AACtB,aAAO,iBAAiB,SAAS,OAAO;AAExC,UAAI;AAEF,cAAM,iBAAiB;AAAA,MACzB,UAAE;AAEA,WAAG,IAAI,WAAW,SAAS;AAC3B,WAAG,IAAI,SAAS,OAAO;AACvB,WAAG,IAAI,SAAS,OAAO;AACvB,eAAO,oBAAoB,SAAS,OAAO;AAAA,MAC7C;AAAA,IACF;AAEA,UAAM,iBAAiB,OAAO,WAAwB;AACpD,UAAI,mBAAkC;AACtC,YAAM,gBAAgB,KAAK,YAAY;AAEvC,YAAM,UAAU,IAAI,6BAAgB,KAAK,KAAK,YAAY,YAAY;AACtE,YAAM,oBAAoB,aAAa,OAAO;AAC9C,YAAM,SAAS,kBAAkB,UAAU;AAE3C,UAAI;AACF,cAAM,eAAe,KAAK;AAE1B,eAAO,CAAC,KAAK,UAAU,CAAC,OAAO,SAAS;AACtC,gBAAM,SAAS,UAAM;AAAA,YACnB,OAAO,KAAK;AAAA,YACZ;AAAA,YACA,MAAM,IAAI,kCAAgB,EAAE,SAAS,wBAAwB,CAAC;AAAA,UAChE;AAEA,cAAI,OAAO,QAAS;AACpB,cAAI,OAAO,KAAM;AAEjB,gBAAM,cAAc,OAAO;AAC3B,kBAAQ,YAAY,MAAM;AAAA,YACxB,KAAK;AACH,iCAAmB,YAAY;AAC/B;AAAA,YACF,KAAK;AACH,oBAAM,aAAa,IAAI,UAAU,OAAO,KAAK,YAAY,OAAO,QAAQ,CAAC;AACzE,yBAAW,SAAS,QAAQ,MAAM,WAAW,MAAM,GAAG;AACpD,8BAAc,kBAAmB,KAAK;AACtC,4BAAY;AAAA,cACd;AACA;AAAA,YACF,KAAK;AACH,yBAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,8BAAc,kBAAmB,KAAK;AACtC,4BAAY;AAAA,cACd;AACA,4BAAc,kBAAmB,IAAI;AACrC,mBAAK,MAAM,IAAI,iBAAiB,aAAa;AAC7C,oBAAM,gBAAgB;AACtB,+BAAiB,QAAQ;AACzB;AAAA,YACF,KAAK;AACH,oBAAM,gBAAgB;AACtB,+BAAiB,QAAQ;AACzB;AAAA,YACF,KAAK;AACH,mBAAK,QAAQ;AAAA,gBACX,EAAE,YAAY;AAAA,gBACd;AAAA,cACF;AACA,oBAAM,gBAAgB;AACtB,+BAAiB;AAAA,gBACf,IAAI,2BAAS,+BAA+B,YAAY,OAAO,EAAE;AAAA,cACnE;AACA;AAAA,YACF;AACE,mBAAK,QAAQ,KAAK,yBAAyB,WAAW;AACtD;AAAA,UACJ;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,YAAI,aAAa,mCAAiB;AAChC,eAAK,QAAQ,KAAK,oDAAoD;AACtE,gBAAM,gBAAgB;AACtB,2BAAiB,OAAO,CAAC;AACzB;AAAA,QACF;AACA,cAAM;AAAA,MACR,UAAE;AACA,eAAO,YAAY;AACnB,YAAI;AACF,gBAAM,kBAAkB,OAAO;AAAA,QACjC,SAAS,GAAG;AACV,eAAK,QAAQ,MAAM,kEAAkE,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF;AAEA,QAAI;AACF,YAAM,KAAK,IAAI,KAAK;AAAA,QAClB,OAAO,OAAkB;AACvB,cAAI;AAGF,kBAAM,gBAAgB,IAAI,gBAAgB;AAC1C,kBAAM,gBAAgB,MAAM,cAAc,MAAM,KAAK,gBAAgB,OAAO,MAAM;AAClF,iBAAK,gBAAgB,OAAO,iBAAiB,SAAS,eAAe,EAAE,MAAM,KAAK,CAAC;AAEnF,kBAAM,QAAQ;AAAA,cACZ,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,gBAAgB,QAAQ;AAAA,gBAChC;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,cACA,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,yBAAyB,IAAI,QAAQ;AAAA,gBAC7C;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,cACA,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,qBAAqB,IAAI,QAAQ;AAAA,gBACzC;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,cACA,kBAAK;AAAA,gBACH,OAAO,eAAe;AACpB,wBAAM,eAAW,6BAAe,cAAc,QAAQ,WAAW,MAAM;AACvE,wBAAM,eAAe,QAAQ;AAAA,gBAC/B;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AAAA,YACF;AAEA,gBAAI;AACF,oBAAM,QAAQ,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;AAAA,YAC9C,UAAE;AAEA,6BAAe,IAAI;AACnB,oBAAM,gBAAgB;AACtB,wBAAM,4BAAc,OAAO,GAAI;AAC/B,mBAAK,gBAAgB,OAAO,oBAAoB,SAAS,aAAa;AAAA,YACxE;AAAA,UACF,SAAS,GAAG;AAEV,gBAAI,aAAa,SAAS,EAAE,SAAS,cAAc;AACjD;AAAA,YACF;AACA,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,QACA;AAAA,UACE,SAAS,KAAK,YAAY;AAAA,QAC5B;AAAA,MACF;AAAA,IACF,SAAS,GAAG;AAEV,UAAI,aAAa,SAAS,EAAE,SAAS,cAAc;AAEjD;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AAEA,YAAM,gBAAgB;AAAA,IACxB;AAAA,EACF;AACF;","names":["import_utils","BaseTTS","BaseSynthesizeStream","tokenizeBasic"]}