@livekit/agents 1.0.39 → 1.0.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/cli.cjs +20 -18
  2. package/dist/cli.cjs.map +1 -1
  3. package/dist/cli.d.ts.map +1 -1
  4. package/dist/cli.js +20 -18
  5. package/dist/cli.js.map +1 -1
  6. package/dist/http_server.cjs +9 -6
  7. package/dist/http_server.cjs.map +1 -1
  8. package/dist/http_server.d.cts +5 -1
  9. package/dist/http_server.d.ts +5 -1
  10. package/dist/http_server.d.ts.map +1 -1
  11. package/dist/http_server.js +9 -6
  12. package/dist/http_server.js.map +1 -1
  13. package/dist/index.cjs +5 -0
  14. package/dist/index.cjs.map +1 -1
  15. package/dist/index.d.cts +1 -0
  16. package/dist/index.d.ts +1 -0
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/index.js +3 -0
  19. package/dist/index.js.map +1 -1
  20. package/dist/inference/stt.cjs +2 -1
  21. package/dist/inference/stt.cjs.map +1 -1
  22. package/dist/inference/stt.d.ts.map +1 -1
  23. package/dist/inference/stt.js +2 -1
  24. package/dist/inference/stt.js.map +1 -1
  25. package/dist/ipc/supervised_proc.cjs +4 -0
  26. package/dist/ipc/supervised_proc.cjs.map +1 -1
  27. package/dist/ipc/supervised_proc.d.cts +1 -0
  28. package/dist/ipc/supervised_proc.d.ts +1 -0
  29. package/dist/ipc/supervised_proc.d.ts.map +1 -1
  30. package/dist/ipc/supervised_proc.js +4 -0
  31. package/dist/ipc/supervised_proc.js.map +1 -1
  32. package/dist/llm/realtime.cjs.map +1 -1
  33. package/dist/llm/realtime.d.cts +5 -1
  34. package/dist/llm/realtime.d.ts +5 -1
  35. package/dist/llm/realtime.d.ts.map +1 -1
  36. package/dist/llm/realtime.js.map +1 -1
  37. package/dist/tokenize/basic/sentence.cjs +3 -3
  38. package/dist/tokenize/basic/sentence.cjs.map +1 -1
  39. package/dist/tokenize/basic/sentence.js +3 -3
  40. package/dist/tokenize/basic/sentence.js.map +1 -1
  41. package/dist/tokenize/tokenizer.test.cjs +3 -1
  42. package/dist/tokenize/tokenizer.test.cjs.map +1 -1
  43. package/dist/tokenize/tokenizer.test.js +3 -1
  44. package/dist/tokenize/tokenizer.test.js.map +1 -1
  45. package/dist/tts/stream_adapter.cjs +15 -1
  46. package/dist/tts/stream_adapter.cjs.map +1 -1
  47. package/dist/tts/stream_adapter.d.ts.map +1 -1
  48. package/dist/tts/stream_adapter.js +15 -1
  49. package/dist/tts/stream_adapter.js.map +1 -1
  50. package/dist/tts/tts.cjs.map +1 -1
  51. package/dist/tts/tts.d.cts +9 -1
  52. package/dist/tts/tts.d.ts +9 -1
  53. package/dist/tts/tts.d.ts.map +1 -1
  54. package/dist/tts/tts.js.map +1 -1
  55. package/dist/types.cjs +3 -0
  56. package/dist/types.cjs.map +1 -1
  57. package/dist/types.d.cts +4 -0
  58. package/dist/types.d.ts +4 -0
  59. package/dist/types.d.ts.map +1 -1
  60. package/dist/types.js +2 -0
  61. package/dist/types.js.map +1 -1
  62. package/dist/voice/agent.cjs +11 -1
  63. package/dist/voice/agent.cjs.map +1 -1
  64. package/dist/voice/agent.d.cts +7 -3
  65. package/dist/voice/agent.d.ts +7 -3
  66. package/dist/voice/agent.d.ts.map +1 -1
  67. package/dist/voice/agent.js +11 -1
  68. package/dist/voice/agent.js.map +1 -1
  69. package/dist/voice/agent_activity.cjs +30 -14
  70. package/dist/voice/agent_activity.cjs.map +1 -1
  71. package/dist/voice/agent_activity.d.cts +1 -0
  72. package/dist/voice/agent_activity.d.ts +1 -0
  73. package/dist/voice/agent_activity.d.ts.map +1 -1
  74. package/dist/voice/agent_activity.js +30 -14
  75. package/dist/voice/agent_activity.js.map +1 -1
  76. package/dist/voice/agent_session.cjs +5 -1
  77. package/dist/voice/agent_session.cjs.map +1 -1
  78. package/dist/voice/agent_session.d.cts +2 -0
  79. package/dist/voice/agent_session.d.ts +2 -0
  80. package/dist/voice/agent_session.d.ts.map +1 -1
  81. package/dist/voice/agent_session.js +5 -1
  82. package/dist/voice/agent_session.js.map +1 -1
  83. package/dist/voice/background_audio.cjs +2 -1
  84. package/dist/voice/background_audio.cjs.map +1 -1
  85. package/dist/voice/background_audio.d.cts +4 -2
  86. package/dist/voice/background_audio.d.ts +4 -2
  87. package/dist/voice/background_audio.d.ts.map +1 -1
  88. package/dist/voice/background_audio.js +2 -1
  89. package/dist/voice/background_audio.js.map +1 -1
  90. package/dist/voice/generation.cjs +58 -5
  91. package/dist/voice/generation.cjs.map +1 -1
  92. package/dist/voice/generation.d.cts +17 -3
  93. package/dist/voice/generation.d.ts +17 -3
  94. package/dist/voice/generation.d.ts.map +1 -1
  95. package/dist/voice/generation.js +63 -6
  96. package/dist/voice/generation.js.map +1 -1
  97. package/dist/voice/index.cjs.map +1 -1
  98. package/dist/voice/index.d.cts +1 -1
  99. package/dist/voice/index.d.ts +1 -1
  100. package/dist/voice/index.d.ts.map +1 -1
  101. package/dist/voice/index.js.map +1 -1
  102. package/dist/voice/io.cjs +22 -2
  103. package/dist/voice/io.cjs.map +1 -1
  104. package/dist/voice/io.d.cts +21 -5
  105. package/dist/voice/io.d.ts +21 -5
  106. package/dist/voice/io.d.ts.map +1 -1
  107. package/dist/voice/io.js +18 -1
  108. package/dist/voice/io.js.map +1 -1
  109. package/dist/voice/room_io/_output.cjs +3 -2
  110. package/dist/voice/room_io/_output.cjs.map +1 -1
  111. package/dist/voice/room_io/_output.d.cts +3 -3
  112. package/dist/voice/room_io/_output.d.ts +3 -3
  113. package/dist/voice/room_io/_output.d.ts.map +1 -1
  114. package/dist/voice/room_io/_output.js +4 -3
  115. package/dist/voice/room_io/_output.js.map +1 -1
  116. package/dist/voice/transcription/synchronizer.cjs +137 -13
  117. package/dist/voice/transcription/synchronizer.cjs.map +1 -1
  118. package/dist/voice/transcription/synchronizer.d.cts +34 -4
  119. package/dist/voice/transcription/synchronizer.d.ts +34 -4
  120. package/dist/voice/transcription/synchronizer.d.ts.map +1 -1
  121. package/dist/voice/transcription/synchronizer.js +141 -14
  122. package/dist/voice/transcription/synchronizer.js.map +1 -1
  123. package/dist/voice/transcription/synchronizer.test.cjs +151 -0
  124. package/dist/voice/transcription/synchronizer.test.cjs.map +1 -0
  125. package/dist/voice/transcription/synchronizer.test.js +150 -0
  126. package/dist/voice/transcription/synchronizer.test.js.map +1 -0
  127. package/dist/worker.cjs +12 -2
  128. package/dist/worker.cjs.map +1 -1
  129. package/dist/worker.d.ts.map +1 -1
  130. package/dist/worker.js +12 -2
  131. package/dist/worker.js.map +1 -1
  132. package/package.json +1 -1
  133. package/src/cli.ts +20 -18
  134. package/src/http_server.ts +18 -6
  135. package/src/index.ts +1 -0
  136. package/src/inference/stt.ts +9 -8
  137. package/src/ipc/supervised_proc.ts +4 -0
  138. package/src/llm/realtime.ts +5 -1
  139. package/src/tokenize/basic/sentence.ts +3 -3
  140. package/src/tokenize/tokenizer.test.ts +4 -0
  141. package/src/tts/stream_adapter.ts +23 -1
  142. package/src/tts/tts.ts +10 -1
  143. package/src/types.ts +5 -0
  144. package/src/voice/agent.ts +19 -4
  145. package/src/voice/agent_activity.ts +38 -13
  146. package/src/voice/agent_session.ts +6 -0
  147. package/src/voice/background_audio.ts +6 -3
  148. package/src/voice/generation.ts +115 -10
  149. package/src/voice/index.ts +1 -1
  150. package/src/voice/io.ts +40 -5
  151. package/src/voice/room_io/_output.ts +6 -5
  152. package/src/voice/transcription/synchronizer.test.ts +206 -0
  153. package/src/voice/transcription/synchronizer.ts +202 -17
  154. package/src/worker.ts +24 -2
package/dist/voice/io.js CHANGED
@@ -2,6 +2,20 @@ import { EventEmitter } from "node:events";
2
2
  import { log } from "../log.js";
3
3
  import { DeferredReadableStream } from "../stream/deferred_stream.js";
4
4
  import { Future } from "../utils.js";
5
+ const TIMED_STRING_SYMBOL = Symbol.for("lk.TimedString");
6
+ function createTimedString(opts) {
7
+ return {
8
+ [TIMED_STRING_SYMBOL]: true,
9
+ text: opts.text,
10
+ startTime: opts.startTime,
11
+ endTime: opts.endTime,
12
+ confidence: opts.confidence,
13
+ startTimeOffset: opts.startTimeOffset
14
+ };
15
+ }
16
+ function isTimedString(value) {
17
+ return typeof value === "object" && value !== null && TIMED_STRING_SYMBOL in value && value[TIMED_STRING_SYMBOL] === true;
18
+ }
5
19
  class AudioInput {
6
20
  deferredStream = new DeferredReadableStream();
7
21
  get stream() {
@@ -249,6 +263,9 @@ export {
249
263
  AgentOutput,
250
264
  AudioInput,
251
265
  AudioOutput,
252
- TextOutput
266
+ TIMED_STRING_SYMBOL,
267
+ TextOutput,
268
+ createTimedString,
269
+ isTimedString
253
270
  };
254
271
  //# sourceMappingURL=io.js.map
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/voice/io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport type { ChatContext } from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport type { ToolContext } from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport type { SpeechEvent } from '../stt/stt.js';\nimport { Future } from '../utils.js';\nimport type { ModelSettings } from './agent.js';\n\nexport type STTNode = (\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<SpeechEvent | string> | null>;\n\nexport type LLMNode = (\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<ChatChunk | string> | null>;\n\nexport type TTSNode = (\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<AudioFrame> | null>;\n\n/**\n *A string with optional start and end timestamps for word-level alignment.\n */\nexport interface TimedString {\n text: string;\n startTime?: number; // seconds\n endTime?: number; // seconds\n confidence?: number;\n startTimeOffset?: number;\n}\n\nexport interface AudioOutputCapabilities {\n /** Whether this output supports pause/resume functionality */\n pause: boolean;\n}\n\nexport abstract class AudioInput {\n protected deferredStream: DeferredReadableStream<AudioFrame> =\n new DeferredReadableStream<AudioFrame>();\n\n get stream(): ReadableStream<AudioFrame> {\n return this.deferredStream.stream;\n }\n\n onAttached(): void {}\n\n onDetached(): void {}\n}\n\nexport abstract class AudioOutput extends EventEmitter {\n static readonly EVENT_PLAYBACK_STARTED = 'playbackStarted';\n static readonly EVENT_PLAYBACK_FINISHED = 'playbackFinished';\n\n private playbackFinishedFuture: Future<void> = new Future();\n private _capturing: boolean = false;\n private playbackFinishedCount: number = 0;\n private playbackSegmentsCount: number = 0;\n private lastPlaybackEvent: PlaybackFinishedEvent = {\n playbackPosition: 0,\n interrupted: false,\n };\n protected logger = log();\n protected readonly capabilities: AudioOutputCapabilities;\n\n constructor(\n public sampleRate?: number,\n protected readonly nextInChain?: AudioOutput,\n capabilities: AudioOutputCapabilities = { pause: false },\n ) {\n super();\n this.capabilities = capabilities;\n\n if (this.nextInChain) {\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, (ev: PlaybackStartedEvent) =>\n this.onPlaybackStarted(ev.createdAt),\n );\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) =>\n this.onPlaybackFinished(ev),\n );\n }\n }\n\n /**\n * Whether this output and all outputs in the chain support pause/resume.\n */\n get canPause(): boolean {\n return this.capabilities.pause && (this.nextInChain?.canPause ?? true);\n }\n\n /**\n * Capture an audio frame for playback, frames can be pushed faster than real-time\n */\n async captureFrame(_frame: AudioFrame): Promise<void> {\n if (!this._capturing) {\n this._capturing = true;\n this.playbackSegmentsCount++;\n }\n }\n\n /**\n * Wait for the past audio segments to finish playing out.\n *\n * @returns The event that was emitted when the audio finished playing out (only the last segment information)\n */\n async waitForPlayout(): Promise<PlaybackFinishedEvent> {\n const target = this.playbackSegmentsCount;\n\n while (this.playbackFinishedCount < target) {\n await this.playbackFinishedFuture.await;\n this.playbackFinishedFuture = new Future();\n }\n\n return this.lastPlaybackEvent;\n }\n\n /**\n * Called when playback actually starts (first frame is sent to output).\n * Developers building audio sinks should call this when the first frame is captured.\n */\n onPlaybackStarted(createdAt: number): void {\n this.emit(AudioOutput.EVENT_PLAYBACK_STARTED, { createdAt } as PlaybackStartedEvent);\n }\n\n /**\n * Developers building audio sinks must call this method when a playback/segment is finished.\n * Segments are segmented by calls to flush() or clearBuffer()\n */\n onPlaybackFinished(options: PlaybackFinishedEvent) {\n if (this.playbackFinishedCount >= this.playbackSegmentsCount) {\n this.logger.warn('playback_finished called more times than playback segments were captured');\n return;\n }\n\n this.lastPlaybackEvent = options;\n this.playbackFinishedCount++;\n this.playbackFinishedFuture.resolve();\n this.emit(AudioOutput.EVENT_PLAYBACK_FINISHED, options);\n }\n\n flush(): void {\n this._capturing = false;\n }\n\n /**\n * Clear the buffer, stopping playback immediately\n */\n abstract clearBuffer(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n\n /**\n * Pause the audio playback\n */\n pause(): void {\n if (this.nextInChain) {\n this.nextInChain.pause();\n }\n }\n\n /**\n * Resume the audio playback\n */\n resume(): void {\n if (this.nextInChain) {\n this.nextInChain.resume();\n }\n }\n}\n\nexport interface PlaybackFinishedEvent {\n /** How much of the audio was played back, in seconds */\n playbackPosition: number;\n /** True if playback was interrupted (clearBuffer() was called) */\n interrupted: boolean;\n /**\n * Transcript synced with playback; may be partial if the audio was interrupted.\n * When undefined, the transcript is not synchronized with the playback.\n */\n synchronizedTranscript?: string;\n}\n\nexport interface PlaybackStartedEvent {\n /** The timestamp (Date.now()) when the playback started */\n createdAt: number;\n}\n\nexport abstract class TextOutput {\n constructor(protected readonly nextInChain?: TextOutput) {}\n\n /**\n * Capture a text segment (Used by the output of LLM nodes)\n */\n abstract captureText(text: string): Promise<void>;\n\n /**\n * Mark the current text segment as complete (e.g LLM generation is complete)\n */\n abstract flush(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n}\n\nexport class AgentInput {\n private _audioStream: AudioInput | null = null;\n // enabled by default\n private _audioEnabled: boolean = true;\n\n constructor(private readonly audioChanged: () => void) {}\n\n setAudioEnabled(enable: boolean): void {\n if (enable === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enable;\n\n if (!this._audioStream) {\n return;\n }\n\n if (enable) {\n this._audioStream.onAttached();\n } else {\n this._audioStream.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get audio(): AudioInput | null {\n return this._audioStream;\n }\n\n set audio(stream: AudioInput | null) {\n this._audioStream = stream;\n this.audioChanged();\n }\n}\n\nexport class AgentOutput {\n private _audioSink: AudioOutput | null = null;\n private _transcriptionSink: TextOutput | null = null;\n private _audioEnabled: boolean = true;\n private _transcriptionEnabled: boolean = true;\n\n constructor(\n private readonly audioChanged: () => void,\n private readonly transcriptionChanged: () => void,\n ) {}\n\n setAudioEnabled(enabled: boolean): void {\n if (enabled === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enabled;\n\n if (!this._audioSink) {\n return;\n }\n\n if (enabled) {\n this._audioSink.onAttached();\n } else {\n this._audioSink.onDetached();\n }\n }\n\n setTranscriptionEnabled(enabled: boolean): void {\n if (enabled === this._transcriptionEnabled) {\n return;\n }\n\n this._transcriptionEnabled = enabled;\n\n if (!this._transcriptionSink) {\n return;\n }\n\n if (enabled) {\n this._transcriptionSink.onAttached();\n } else {\n this._transcriptionSink.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get transcriptionEnabled(): boolean {\n return this._transcriptionEnabled;\n }\n\n get audio(): AudioOutput | null {\n return this._audioSink;\n }\n\n set audio(sink: AudioOutput | null) {\n if (sink === this._audioSink) {\n return;\n }\n\n if (this._audioSink) {\n this._audioSink.onDetached();\n }\n\n this._audioSink = sink;\n this.audioChanged();\n\n if (this._audioSink) {\n this._audioSink.onAttached();\n }\n }\n\n get transcription(): TextOutput | null {\n return this._transcriptionSink;\n }\n\n set transcription(sink: TextOutput | null) {\n if (sink === this._transcriptionSink) {\n return;\n }\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onDetached();\n }\n\n this._transcriptionSink = sink;\n this.transcriptionChanged();\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onAttached();\n }\n }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAK7B,SAAS,WAAW;AACpB,SAAS,8BAA8B;AAEvC,SAAS,cAAc;AAmChB,MAAe,WAAW;AAAA,EACrB,iBACR,IAAI,uBAAmC;AAAA,EAEzC,IAAI,SAAqC;AACvC,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,aAAmB;AAAA,EAAC;AAAA,EAEpB,aAAmB;AAAA,EAAC;AACtB;AAEO,MAAe,oBAAoB,aAAa;AAAA,EAerD,YACS,YACY,aACnB,eAAwC,EAAE,OAAO,MAAM,GACvD;AACA,UAAM;AAJC;AACY;AAInB,SAAK,eAAe;AAEpB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAwB,CAAC,OACvD,KAAK,kBAAkB,GAAG,SAAS;AAAA,MACrC;AACA,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAyB,CAAC,OACxD,KAAK,mBAAmB,EAAE;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAAA,EA9BA,OAAgB,yBAAyB;AAAA,EACzC,OAAgB,0BAA0B;AAAA,EAElC,yBAAuC,IAAI,OAAO;AAAA,EAClD,aAAsB;AAAA,EACtB,wBAAgC;AAAA,EAChC,wBAAgC;AAAA,EAChC,oBAA2C;AAAA,IACjD,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf;AAAA,EACU,SAAS,IAAI;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA,EAuBnB,IAAI,WAAoB;AAhG1B;AAiGI,WAAO,KAAK,aAAa,YAAU,UAAK,gBAAL,mBAAkB,aAAY;AAAA,EACnE;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aAAa,QAAmC;AACpD,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa;AAClB,WAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,iBAAiD;AACrD,UAAM,SAAS,KAAK;AAEpB,WAAO,KAAK,wBAAwB,QAAQ;AAC1C,YAAM,KAAK,uBAAuB;AAClC,WAAK,yBAAyB,IAAI,OAAO;AAAA,IAC3C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAkB,WAAyB;AACzC,SAAK,KAAK,YAAY,wBAAwB,EAAE,UAAU,CAAyB;AAAA,EACrF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,mBAAmB,SAAgC;AACjD,QAAI,KAAK,yBAAyB,KAAK,uBAAuB;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAC3F;AAAA,IACF;AAEA,SAAK,oBAAoB;AACzB,SAAK;AACL,SAAK,uBAAuB,QAAQ;AACpC,SAAK,KAAK,YAAY,yBAAyB,OAAO;AAAA,EACxD;AAAA,EAEA,QAAc;AACZ,SAAK,aAAa;AAAA,EACpB;AAAA,EAOA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,MAAM;AAAA,IACzB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,SAAe;AACb,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,OAAO;AAAA,IAC1B;AAAA,EACF;AACF;AAmBO,MAAe,WAAW;AAAA,EAC/B,YAA+B,aAA0B;AAA1B;AAAA,EAA2B;AAAA,EAY1D,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AACF;AAEO,MAAM,WAAW;AAAA,EAKtB,YAA6B,cAA0B;AAA1B;AAAA,EAA2B;AAAA,EAJhD,eAAkC;AAAA;AAAA,EAElC,gBAAyB;AAAA,EAIjC,gBAAgB,QAAuB;AACrC,QAAI,WAAW,KAAK,eAAe;AACjC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,QAAI,QAAQ;AACV,WAAK,aAAa,WAAW;AAAA,IAC/B,OAAO;AACL,WAAK,aAAa,WAAW;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,QAA2B;AACnC,SAAK,eAAe;AACpB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,YAAY;AAAA,EAMvB,YACmB,cACA,sBACjB;AAFiB;AACA;AAAA,EAChB;AAAA,EARK,aAAiC;AAAA,EACjC,qBAAwC;AAAA,EACxC,gBAAyB;AAAA,EACzB,wBAAiC;AAAA,EAOzC,gBAAgB,SAAwB;AACtC,QAAI,YAAY,KAAK,eAAe;AAClC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,WAAW,WAAW;AAAA,IAC7B,OAAO;AACL,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,wBAAwB,SAAwB;AAC9C,QAAI,YAAY,KAAK,uBAAuB;AAC1C;AAAA,IACF;AAEA,SAAK,wBAAwB;AAE7B,QAAI,CAAC,KAAK,oBAAoB;AAC5B;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,mBAAmB,WAAW;AAAA,IACrC,OAAO;AACL,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,uBAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,MAA0B;AAClC,QAAI,SAAS,KAAK,YAAY;AAC5B;AAAA,IACF;AAEA,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAEA,SAAK,aAAa;AAClB,SAAK,aAAa;AAElB,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,IAAI,gBAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAc,MAAyB;AACzC,QAAI,SAAS,KAAK,oBAAoB;AACpC;AAAA,IACF;AAEA,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAEA,SAAK,qBAAqB;AAC1B,SAAK,qBAAqB;AAE1B,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AACF;","names":[]}
1
+ {"version":3,"sources":["../../src/voice/io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport type { ChatContext } from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport type { ToolContext } from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport type { SpeechEvent } from '../stt/stt.js';\nimport { Future } from '../utils.js';\nimport type { ModelSettings } from './agent.js';\n\nexport type STTNode = (\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<SpeechEvent | string> | null>;\n\nexport type LLMNode = (\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<ChatChunk | string> | null>;\n\nexport type TTSNode = (\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<AudioFrame> | null>;\n\n/**\n * Symbol used to identify TimedString objects.\n */\nexport const TIMED_STRING_SYMBOL = Symbol.for('lk.TimedString');\n\n/**\n * A string with optional start and end timestamps for word-level alignment.\n */\nexport interface TimedString {\n readonly [TIMED_STRING_SYMBOL]: true;\n text: string;\n startTime?: number; // seconds\n endTime?: number; // seconds\n confidence?: number;\n startTimeOffset?: number;\n}\n\n/**\n * Factory function to create a TimedString object.\n */\nexport function createTimedString(opts: {\n text: string;\n startTime?: number;\n endTime?: number;\n confidence?: number;\n startTimeOffset?: number;\n}): TimedString {\n return {\n [TIMED_STRING_SYMBOL]: true,\n text: opts.text,\n startTime: opts.startTime,\n endTime: opts.endTime,\n confidence: opts.confidence,\n startTimeOffset: opts.startTimeOffset,\n };\n}\n\n/**\n * Type guard to check if a value is a TimedString.\n */\nexport function isTimedString(value: unknown): value is TimedString {\n return (\n typeof value === 'object' &&\n value !== null &&\n TIMED_STRING_SYMBOL in value &&\n (value as TimedString)[TIMED_STRING_SYMBOL] === true\n );\n}\n\nexport interface AudioOutputCapabilities {\n /** Whether this output supports pause/resume functionality */\n pause: boolean;\n}\n\nexport abstract class AudioInput {\n protected deferredStream: DeferredReadableStream<AudioFrame> =\n new DeferredReadableStream<AudioFrame>();\n\n get stream(): ReadableStream<AudioFrame> {\n return this.deferredStream.stream;\n }\n\n onAttached(): void {}\n\n onDetached(): void {}\n}\n\nexport abstract class AudioOutput extends EventEmitter {\n static readonly EVENT_PLAYBACK_STARTED = 'playbackStarted';\n static readonly EVENT_PLAYBACK_FINISHED = 'playbackFinished';\n\n private playbackFinishedFuture: Future<void> = new Future();\n private _capturing: boolean = false;\n private playbackFinishedCount: number = 0;\n private playbackSegmentsCount: number = 0;\n private lastPlaybackEvent: PlaybackFinishedEvent = {\n playbackPosition: 0,\n interrupted: false,\n };\n protected logger = log();\n protected readonly capabilities: AudioOutputCapabilities;\n\n constructor(\n public sampleRate?: number,\n protected readonly nextInChain?: AudioOutput,\n capabilities: AudioOutputCapabilities = { pause: false },\n ) {\n super();\n this.capabilities = capabilities;\n\n if (this.nextInChain) {\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, (ev: PlaybackStartedEvent) =>\n this.onPlaybackStarted(ev.createdAt),\n );\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) =>\n this.onPlaybackFinished(ev),\n );\n }\n }\n\n /**\n * Whether this output and all outputs in the chain support pause/resume.\n */\n get canPause(): boolean {\n return this.capabilities.pause && (this.nextInChain?.canPause ?? true);\n }\n\n /**\n * Capture an audio frame for playback, frames can be pushed faster than real-time\n */\n async captureFrame(_frame: AudioFrame): Promise<void> {\n if (!this._capturing) {\n this._capturing = true;\n this.playbackSegmentsCount++;\n }\n }\n\n /**\n * Wait for the past audio segments to finish playing out.\n *\n * @returns The event that was emitted when the audio finished playing out (only the last segment information)\n */\n async waitForPlayout(): Promise<PlaybackFinishedEvent> {\n const target = this.playbackSegmentsCount;\n\n while (this.playbackFinishedCount < target) {\n await this.playbackFinishedFuture.await;\n this.playbackFinishedFuture = new Future();\n }\n\n return this.lastPlaybackEvent;\n }\n\n /**\n * Called when playback actually starts (first frame is sent to output).\n * Developers building audio sinks should call this when the first frame is captured.\n */\n onPlaybackStarted(createdAt: number): void {\n this.emit(AudioOutput.EVENT_PLAYBACK_STARTED, { createdAt } as PlaybackStartedEvent);\n }\n\n /**\n * Developers building audio sinks must call this method when a playback/segment is finished.\n * Segments are segmented by calls to flush() or clearBuffer()\n */\n onPlaybackFinished(options: PlaybackFinishedEvent) {\n if (this.playbackFinishedCount >= this.playbackSegmentsCount) {\n this.logger.warn('playback_finished called more times than playback segments were captured');\n return;\n }\n\n this.lastPlaybackEvent = options;\n this.playbackFinishedCount++;\n this.playbackFinishedFuture.resolve();\n this.emit(AudioOutput.EVENT_PLAYBACK_FINISHED, options);\n }\n\n flush(): void {\n this._capturing = false;\n }\n\n /**\n * Clear the buffer, stopping playback immediately\n */\n abstract clearBuffer(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n\n /**\n * Pause the audio playback\n */\n pause(): void {\n if (this.nextInChain) {\n this.nextInChain.pause();\n }\n }\n\n /**\n * Resume the audio playback\n */\n resume(): void {\n if (this.nextInChain) {\n this.nextInChain.resume();\n }\n }\n}\n\nexport interface PlaybackFinishedEvent {\n /** How much of the audio was played back, in seconds */\n playbackPosition: number;\n /** True if playback was interrupted (clearBuffer() was called) */\n interrupted: boolean;\n /**\n * Transcript synced with playback; may be partial if the audio was interrupted.\n * When undefined, the transcript is not synchronized with the playback.\n */\n synchronizedTranscript?: string;\n}\n\nexport interface PlaybackStartedEvent {\n /** The timestamp (Date.now()) when the playback started */\n createdAt: number;\n}\n\nexport abstract class TextOutput {\n constructor(protected readonly nextInChain?: TextOutput) {}\n\n abstract captureText(text: string | TimedString): Promise<void>;\n\n /**\n * Mark the current text segment as complete (e.g LLM generation is complete)\n */\n abstract flush(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n}\n\nexport class AgentInput {\n private _audioStream: AudioInput | null = null;\n // enabled by default\n private _audioEnabled: boolean = true;\n\n constructor(private readonly audioChanged: () => void) {}\n\n setAudioEnabled(enable: boolean): void {\n if (enable === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enable;\n\n if (!this._audioStream) {\n return;\n }\n\n if (enable) {\n this._audioStream.onAttached();\n } else {\n this._audioStream.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get audio(): AudioInput | null {\n return this._audioStream;\n }\n\n set audio(stream: AudioInput | null) {\n this._audioStream = stream;\n this.audioChanged();\n }\n}\n\nexport class AgentOutput {\n private _audioSink: AudioOutput | null = null;\n private _transcriptionSink: TextOutput | null = null;\n private _audioEnabled: boolean = true;\n private _transcriptionEnabled: boolean = true;\n\n constructor(\n private readonly audioChanged: () => void,\n private readonly transcriptionChanged: () => void,\n ) {}\n\n setAudioEnabled(enabled: boolean): void {\n if (enabled === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enabled;\n\n if (!this._audioSink) {\n return;\n }\n\n if (enabled) {\n this._audioSink.onAttached();\n } else {\n this._audioSink.onDetached();\n }\n }\n\n setTranscriptionEnabled(enabled: boolean): void {\n if (enabled === this._transcriptionEnabled) {\n return;\n }\n\n this._transcriptionEnabled = enabled;\n\n if (!this._transcriptionSink) {\n return;\n }\n\n if (enabled) {\n this._transcriptionSink.onAttached();\n } else {\n this._transcriptionSink.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get transcriptionEnabled(): boolean {\n return this._transcriptionEnabled;\n }\n\n get audio(): AudioOutput | null {\n return this._audioSink;\n }\n\n set audio(sink: AudioOutput | null) {\n if (sink === this._audioSink) {\n return;\n }\n\n if (this._audioSink) {\n this._audioSink.onDetached();\n }\n\n this._audioSink = sink;\n this.audioChanged();\n\n if (this._audioSink) {\n this._audioSink.onAttached();\n }\n }\n\n get transcription(): TextOutput | null {\n return this._transcriptionSink;\n }\n\n set transcription(sink: TextOutput | null) {\n if (sink === this._transcriptionSink) {\n return;\n }\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onDetached();\n }\n\n this._transcriptionSink = sink;\n this.transcriptionChanged();\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onAttached();\n }\n }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAK7B,SAAS,WAAW;AACpB,SAAS,8BAA8B;AAEvC,SAAS,cAAc;AAsBhB,MAAM,sBAAsB,OAAO,IAAI,gBAAgB;AAiBvD,SAAS,kBAAkB,MAMlB;AACd,SAAO;AAAA,IACL,CAAC,mBAAmB,GAAG;AAAA,IACvB,MAAM,KAAK;AAAA,IACX,WAAW,KAAK;AAAA,IAChB,SAAS,KAAK;AAAA,IACd,YAAY,KAAK;AAAA,IACjB,iBAAiB,KAAK;AAAA,EACxB;AACF;AAKO,SAAS,cAAc,OAAsC;AAClE,SACE,OAAO,UAAU,YACjB,UAAU,QACV,uBAAuB,SACtB,MAAsB,mBAAmB,MAAM;AAEpD;AAOO,MAAe,WAAW;AAAA,EACrB,iBACR,IAAI,uBAAmC;AAAA,EAEzC,IAAI,SAAqC;AACvC,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,aAAmB;AAAA,EAAC;AAAA,EAEpB,aAAmB;AAAA,EAAC;AACtB;AAEO,MAAe,oBAAoB,aAAa;AAAA,EAerD,YACS,YACY,aACnB,eAAwC,EAAE,OAAO,MAAM,GACvD;AACA,UAAM;AAJC;AACY;AAInB,SAAK,eAAe;AAEpB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAwB,CAAC,OACvD,KAAK,kBAAkB,GAAG,SAAS;AAAA,MACrC;AACA,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAyB,CAAC,OACxD,KAAK,mBAAmB,EAAE;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAAA,EA9BA,OAAgB,yBAAyB;AAAA,EACzC,OAAgB,0BAA0B;AAAA,EAElC,yBAAuC,IAAI,OAAO;AAAA,EAClD,aAAsB;AAAA,EACtB,wBAAgC;AAAA,EAChC,wBAAgC;AAAA,EAChC,oBAA2C;AAAA,IACjD,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf;AAAA,EACU,SAAS,IAAI;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA,EAuBnB,IAAI,WAAoB;AAtI1B;AAuII,WAAO,KAAK,aAAa,YAAU,UAAK,gBAAL,mBAAkB,aAAY;AAAA,EACnE;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aAAa,QAAmC;AACpD,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa;AAClB,WAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,iBAAiD;AACrD,UAAM,SAAS,KAAK;AAEpB,WAAO,KAAK,wBAAwB,QAAQ;AAC1C,YAAM,KAAK,uBAAuB;AAClC,WAAK,yBAAyB,IAAI,OAAO;AAAA,IAC3C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAkB,WAAyB;AACzC,SAAK,KAAK,YAAY,wBAAwB,EAAE,UAAU,CAAyB;AAAA,EACrF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,mBAAmB,SAAgC;AACjD,QAAI,KAAK,yBAAyB,KAAK,uBAAuB;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAC3F;AAAA,IACF;AAEA,SAAK,oBAAoB;AACzB,SAAK;AACL,SAAK,uBAAuB,QAAQ;AACpC,SAAK,KAAK,YAAY,yBAAyB,OAAO;AAAA,EACxD;AAAA,EAEA,QAAc;AACZ,SAAK,aAAa;AAAA,EACpB;AAAA,EAOA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,MAAM;AAAA,IACzB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,SAAe;AACb,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,OAAO;AAAA,IAC1B;AAAA,EACF;AACF;AAmBO,MAAe,WAAW;AAAA,EAC/B,YAA+B,aAA0B;AAA1B;AAAA,EAA2B;AAAA,EAS1D,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AACF;AAEO,MAAM,WAAW;AAAA,EAKtB,YAA6B,cAA0B;AAA1B;AAAA,EAA2B;AAAA,EAJhD,eAAkC;AAAA;AAAA,EAElC,gBAAyB;AAAA,EAIjC,gBAAgB,QAAuB;AACrC,QAAI,WAAW,KAAK,eAAe;AACjC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,QAAI,QAAQ;AACV,WAAK,aAAa,WAAW;AAAA,IAC/B,OAAO;AACL,WAAK,aAAa,WAAW;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,QAA2B;AACnC,SAAK,eAAe;AACpB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,YAAY;AAAA,EAMvB,YACmB,cACA,sBACjB;AAFiB;AACA;AAAA,EAChB;AAAA,EARK,aAAiC;AAAA,EACjC,qBAAwC;AAAA,EACxC,gBAAyB;AAAA,EACzB,wBAAiC;AAAA,EAOzC,gBAAgB,SAAwB;AACtC,QAAI,YAAY,KAAK,eAAe;AAClC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,WAAW,WAAW;AAAA,IAC7B,OAAO;AACL,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,wBAAwB,SAAwB;AAC9C,QAAI,YAAY,KAAK,uBAAuB;AAC1C;AAAA,IACF;AAEA,SAAK,wBAAwB;AAE7B,QAAI,CAAC,KAAK,oBAAoB;AAC5B;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,mBAAmB,WAAW;AAAA,IACrC,OAAO;AACL,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,uBAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,MAA0B;AAClC,QAAI,SAAS,KAAK,YAAY;AAC5B;AAAA,IACF;AAEA,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAEA,SAAK,aAAa;AAClB,SAAK,aAAa;AAElB,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,IAAI,gBAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAc,MAAyB;AACzC,QAAI,SAAS,KAAK,oBAAoB;AACpC;AAAA,IACF;AAEA,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAEA,SAAK,qBAAqB;AAC1B,SAAK,qBAAqB;AAE1B,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AACF;","names":[]}
@@ -88,8 +88,9 @@ class BaseParticipantTranscriptionOutput extends import_io.TextOutput {
88
88
  if (!this.participantIdentity) {
89
89
  return;
90
90
  }
91
- this.latestText = text;
92
- await this.handleCaptureText(text);
91
+ const textStr = (0, import_io.isTimedString)(text) ? text.text : text;
92
+ this.latestText = textStr;
93
+ await this.handleCaptureText(textStr);
93
94
  }
94
95
  flush() {
95
96
  if (!this.participantIdentity || !this.capturing) {
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string) {\n if (!this.participantIdentity) {\n return;\n }\n\n this.latestText = text;\n await this.handleCaptureText(text);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAYO;AACP,uBAKO;AACP,iBAAoB;AACpB,mBAAwC;AACxC,gBAAwC;AACxC,2BAAsC;AAEtC,MAAe,2CAA2C,qBAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,aAAS,gBAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,0BAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,0BAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,cAAU,4CAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,eAAO,wBAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,IAAI;AAAA,EACnC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,8CAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,iDAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,mDAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,8CAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,iDAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAvQ7F;AAwQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,qBAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,sBAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,oBAAO;AAAA,EACzC,oBAAkC,IAAI,oBAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,4BAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,oBAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,oBAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAxalD;AAyaI,UAAM,QAAQ,gCAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oCAAoB,EAAE,QAAQ,4BAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
1
+ {"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string | TimedString) {\n if (!this.participantIdentity) {\n return;\n }\n\n const textStr = isTimedString(text) ? text.text : text;\n this.latestText = textStr;\n await this.handleCaptureText(textStr);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string | TimedString) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAYO;AACP,uBAKO;AACP,iBAAoB;AACpB,mBAAwC;AACxC,gBAAyE;AACzE,2BAAsC;AAEtC,MAAe,2CAA2C,qBAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,aAAS,gBAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,0BAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,0BAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,cAAU,4CAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,eAAO,wBAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,UAAM,cAAU,yBAAc,IAAI,IAAI,KAAK,OAAO;AAClD,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,OAAO;AAAA,EACtC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,8CAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,iDAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,mDAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,8CAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,iDAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAxQ7F;AAyQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,qBAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,sBAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,oBAAO;AAAA,EACzC,oBAAkC,IAAI,oBAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,4BAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,oBAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,oBAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAzalD;AA0aI,UAAM,QAAQ,gCAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oCAAoB,EAAE,QAAQ,4BAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
@@ -1,6 +1,6 @@
1
1
  import type { RemoteParticipant } from '@livekit/rtc-node';
2
2
  import { type AudioFrame, type LocalTrackPublication, type Participant, type RemoteTrackPublication, type Room, TrackPublishOptions } from '@livekit/rtc-node';
3
- import { AudioOutput, TextOutput } from '../io.js';
3
+ import { AudioOutput, TextOutput, type TimedString } from '../io.js';
4
4
  declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
5
5
  protected room: Room;
6
6
  protected isDeltaStream: boolean;
@@ -16,7 +16,7 @@ declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
16
16
  protected onLocalTrackPublished: (track: LocalTrackPublication) => void;
17
17
  protected generateCurrentId(): string;
18
18
  protected resetState(): void;
19
- captureText(text: string): Promise<void>;
19
+ captureText(text: string | TimedString): Promise<void>;
20
20
  flush(): void;
21
21
  protected abstract handleCaptureText(text: string): Promise<void>;
22
22
  protected abstract handleFlush(): void;
@@ -41,7 +41,7 @@ export declare class ParalellTextOutput extends TextOutput {
41
41
  /** @internal */
42
42
  _sinks: TextOutput[];
43
43
  constructor(sinks: TextOutput[], nextInChain?: TextOutput);
44
- captureText(text: string): Promise<void>;
44
+ captureText(text: string | TimedString): Promise<void>;
45
45
  flush(): void;
46
46
  }
47
47
  export interface AudioOutputOptions {
@@ -1,6 +1,6 @@
1
1
  import type { RemoteParticipant } from '@livekit/rtc-node';
2
2
  import { type AudioFrame, type LocalTrackPublication, type Participant, type RemoteTrackPublication, type Room, TrackPublishOptions } from '@livekit/rtc-node';
3
- import { AudioOutput, TextOutput } from '../io.js';
3
+ import { AudioOutput, TextOutput, type TimedString } from '../io.js';
4
4
  declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
5
5
  protected room: Room;
6
6
  protected isDeltaStream: boolean;
@@ -16,7 +16,7 @@ declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
16
16
  protected onLocalTrackPublished: (track: LocalTrackPublication) => void;
17
17
  protected generateCurrentId(): string;
18
18
  protected resetState(): void;
19
- captureText(text: string): Promise<void>;
19
+ captureText(text: string | TimedString): Promise<void>;
20
20
  flush(): void;
21
21
  protected abstract handleCaptureText(text: string): Promise<void>;
22
22
  protected abstract handleFlush(): void;
@@ -41,7 +41,7 @@ export declare class ParalellTextOutput extends TextOutput {
41
41
  /** @internal */
42
42
  _sinks: TextOutput[];
43
43
  constructor(sinks: TextOutput[], nextInChain?: TextOutput);
44
- captureText(text: string): Promise<void>;
44
+ captureText(text: string | TimedString): Promise<void>;
45
45
  flush(): void;
46
46
  }
47
47
  export interface AudioOutputOptions {
@@ -1 +1 @@
1
- {"version":3,"file":"_output.d.ts","sourceRoot":"","sources":["../../../src/voice/room_io/_output.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,sBAAsB,EAC3B,KAAK,IAAI,EAGT,mBAAmB,EAEpB,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGnD,uBAAe,kCAAmC,SAAQ,UAAU;IAClE,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC;IACrB,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAQ;IACpD,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,SAAS,EAAE,OAAO,CAAS;IACrC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAM;IAClC,SAAS,CAAC,SAAS,EAAE,MAAM,CAA4B;IACvD,SAAS,CAAC,MAAM,wBAAS;gBAEb,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAWxF,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAqBvD,SAAS,CAAC,gBAAgB,UAAW,sBAAsB,eAAe,iBAAiB,UAUzF;IAEF,SAAS,CAAC,qBAAqB,UAAW,qBAAqB,UAU7D;IAEF,SAAS,CAAC,iBAAiB,IAAI,MAAM;IAIrC,SAAS,CAAC,UAAU;IAMd,WAAW,CAAC,IAAI,EAAE,MAAM;IAS9B,KAAK;IASL,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IACjE,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI;CACvC;AAED,qBAAa,8BAA+B,SAAQ,kCAAkC;IACpF,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,SAAS,CAA2B;cAE5B,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B9D,SAAS,CAAC,WAAW;YAMP,gBAAgB;YA0BhB,aAAa;CAkC5B;AAED,qBAAa,oCAAqC,SAAQ,kCAAkC;IAC1F,OAAO,CAAC,UAAU,CAAc;IAChC,OAAO,CAAC,SAAS,CAA8B;cAE/B,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB9D,SAAS,CAAC,WAAW;IASf,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW;IAsBzF,SAAS,CAAC,UAAU;CAIrB;AAED,qBAAa,kBAAmB,SAAQ,UAAU;IAChD,gBAAgB;IAChB,MAAM,EAAE,UAAU,EAAE,CAAC;gBAET,KAAK,EAAE,UAAU,EAAE,EAAE,WAAW,CAAC,EAAE,UAAU;IAKnD,WAAW,CAAC,IAAI,EAAE,MAAM;IAI9B,KAAK;CAKN;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,mBAAmB,CAAC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,qBAAa,sBAAuB,SAAQ,WAAW;IACrD,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,WAAW,CAAC,CAAwB;IAC5C,OAAO,CAAC,SAAS,CAAC,CAAa;IAE/B,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,iBAAiB,CAA8B;IACvD,OAAO,CAAC,iBAAiB,CAAkB;gBAE/B,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,kBAAkB;IAOnD,IAAI,UAAU,IAAI,OAAO,CAExB;IAEK,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;YAetC,kBAAkB;IAsChC;;OAEG;IACH,KAAK,IAAI,IAAI;IAeb,WAAW,IAAI,IAAI;YAQL,YAAY;IAkBpB,KAAK;CAIZ"}
1
+ {"version":3,"file":"_output.d.ts","sourceRoot":"","sources":["../../../src/voice/room_io/_output.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,sBAAsB,EAC3B,KAAK,IAAI,EAGT,mBAAmB,EAEpB,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,KAAK,WAAW,EAAiB,MAAM,UAAU,CAAC;AAGpF,uBAAe,kCAAmC,SAAQ,UAAU;IAClE,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC;IACrB,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAQ;IACpD,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,SAAS,EAAE,OAAO,CAAS;IACrC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAM;IAClC,SAAS,CAAC,SAAS,EAAE,MAAM,CAA4B;IACvD,SAAS,CAAC,MAAM,wBAAS;gBAEb,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAWxF,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAqBvD,SAAS,CAAC,gBAAgB,UAAW,sBAAsB,eAAe,iBAAiB,UAUzF;IAEF,SAAS,CAAC,qBAAqB,UAAW,qBAAqB,UAU7D;IAEF,SAAS,CAAC,iBAAiB,IAAI,MAAM;IAIrC,SAAS,CAAC,UAAU;IAMd,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW;IAU5C,KAAK;IASL,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IACjE,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI;CACvC;AAED,qBAAa,8BAA+B,SAAQ,kCAAkC;IACpF,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,SAAS,CAA2B;cAE5B,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B9D,SAAS,CAAC,WAAW;YAMP,gBAAgB;YA0BhB,aAAa;CAkC5B;AAED,qBAAa,oCAAqC,SAAQ,kCAAkC;IAC1F,OAAO,CAAC,UAAU,CAAc;IAChC,OAAO,CAAC,SAAS,CAA8B;cAE/B,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB9D,SAAS,CAAC,WAAW;IASf,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW;IAsBzF,SAAS,CAAC,UAAU;CAIrB;AAED,qBAAa,kBAAmB,SAAQ,UAAU;IAChD,gBAAgB;IAChB,MAAM,EAAE,UAAU,EAAE,CAAC;gBAET,KAAK,EAAE,UAAU,EAAE,EAAE,WAAW,CAAC,EAAE,UAAU;IAKnD,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW;IAI5C,KAAK;CAKN;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,mBAAmB,CAAC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,qBAAa,sBAAuB,SAAQ,WAAW;IACrD,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,WAAW,CAAC,CAAwB;IAC5C,OAAO,CAAC,SAAS,CAAC,CAAa;IAE/B,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,iBAAiB,CAA8B;IACvD,OAAO,CAAC,iBAAiB,CAAkB;gBAE/B,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,kBAAkB;IAOnD,IAAI,UAAU,IAAI,OAAO,CAExB;IAEK,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;YAetC,kBAAkB;IAsChC;;OAEG;IACH,KAAK,IAAI,IAAI;IAeb,WAAW,IAAI,IAAI;YAQL,YAAY;IAkBpB,KAAK;CAIZ"}
@@ -13,7 +13,7 @@ import {
13
13
  } from "../../constants.js";
14
14
  import { log } from "../../log.js";
15
15
  import { Future, Task, shortuuid } from "../../utils.js";
16
- import { AudioOutput, TextOutput } from "../io.js";
16
+ import { AudioOutput, TextOutput, isTimedString } from "../io.js";
17
17
  import { findMicrophoneTrackId } from "../transcription/index.js";
18
18
  class BaseParticipantTranscriptionOutput extends TextOutput {
19
19
  room;
@@ -73,8 +73,9 @@ class BaseParticipantTranscriptionOutput extends TextOutput {
73
73
  if (!this.participantIdentity) {
74
74
  return;
75
75
  }
76
- this.latestText = text;
77
- await this.handleCaptureText(text);
76
+ const textStr = isTimedString(text) ? text.text : text;
77
+ this.latestText = textStr;
78
+ await this.handleCaptureText(textStr);
78
79
  }
79
80
  flush() {
80
81
  if (!this.participantIdentity || !this.capturing) {
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string) {\n if (!this.participantIdentity) {\n return;\n }\n\n this.latestText = text;\n await this.handleCaptureText(text);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":"AAIA;AAAA,EAEE;AAAA,EACA;AAAA,EAKA;AAAA,EAEA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,WAAW;AACpB,SAAS,QAAQ,MAAM,iBAAiB;AACxC,SAAS,aAAa,kBAAkB;AACxC,SAAS,6BAA6B;AAEtC,MAAe,2CAA2C,WAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,SAAS,IAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,UAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,UAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,UAAU,sBAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,WAAO,UAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,IAAI;AAAA,EACnC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,6BAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,gCAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,kCAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,6BAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,gCAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAvQ7F;AAwQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,WAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,YAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,OAAO;AAAA,EACzC,oBAAkC,IAAI,OAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,YAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,OAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,OAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAxalD;AAyaI,UAAM,QAAQ,gBAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oBAAoB,EAAE,QAAQ,YAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
1
+ {"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string | TimedString) {\n if (!this.participantIdentity) {\n return;\n }\n\n const textStr = isTimedString(text) ? text.text : text;\n this.latestText = textStr;\n await this.handleCaptureText(textStr);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string | TimedString) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":"AAIA;AAAA,EAEE;AAAA,EACA;AAAA,EAKA;AAAA,EAEA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,WAAW;AACpB,SAAS,QAAQ,MAAM,iBAAiB;AACxC,SAAS,aAAa,YAA8B,qBAAqB;AACzE,SAAS,6BAA6B;AAEtC,MAAe,2CAA2C,WAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,SAAS,IAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,UAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,UAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,UAAU,sBAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,WAAO,UAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,UAAM,UAAU,cAAc,IAAI,IAAI,KAAK,OAAO;AAClD,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,OAAO;AAAA,EACtC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,6BAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,gCAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,kCAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,6BAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,gCAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAxQ7F;AAyQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,WAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,YAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,OAAO;AAAA,EACzC,oBAAkC,IAAI,OAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,YAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,OAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,OAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAzalD;AA0aI,UAAM,QAAQ,gBAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oBAAoB,EAAE,QAAQ,YAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
@@ -18,6 +18,7 @@ var __copyProps = (to, from, except, desc) => {
18
18
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
19
  var synchronizer_exports = {};
20
20
  __export(synchronizer_exports, {
21
+ SpeakingRateData: () => SpeakingRateData,
21
22
  TranscriptionSynchronizer: () => TranscriptionSynchronizer,
22
23
  defaultTextSyncOptions: () => defaultTextSyncOptions
23
24
  });
@@ -28,6 +29,78 @@ var import_tokenize = require("../../tokenize/index.cjs");
28
29
  var import_utils = require("../../utils.cjs");
29
30
  var import_io = require("../io.cjs");
30
31
  const STANDARD_SPEECH_RATE = 3.83;
32
+ class SpeakingRateData {
33
+ /** Timestamps of the speaking rate. */
34
+ timestamps = [];
35
+ /** Speed at the timestamp. */
36
+ speakingRate = [];
37
+ /** Accumulated speaking units up to the timestamp. */
38
+ speakIntegrals = [];
39
+ /** Buffer for text without timing annotations yet. */
40
+ textBuffer = [];
41
+ /**
42
+ * Add by speaking rate estimation.
43
+ */
44
+ addByRate(timestamp, speakingRate) {
45
+ const integral = this.speakIntegrals.length > 0 ? this.speakIntegrals[this.speakIntegrals.length - 1] : 0;
46
+ const dt = timestamp - this.pushedDuration;
47
+ const newIntegral = integral + speakingRate * dt;
48
+ this.timestamps.push(timestamp);
49
+ this.speakingRate.push(speakingRate);
50
+ this.speakIntegrals.push(newIntegral);
51
+ }
52
+ /**
53
+ * Add annotation from TimedString with start_time/end_time.
54
+ */
55
+ addByAnnotation(text, startTime, endTime) {
56
+ if (startTime !== void 0) {
57
+ const integral = this.speakIntegrals.length > 0 ? this.speakIntegrals[this.speakIntegrals.length - 1] : 0;
58
+ const dt = startTime - this.pushedDuration;
59
+ const textLen = this.textBuffer.reduce((sum, t) => sum + t.length, 0);
60
+ const newIntegral = integral + textLen;
61
+ const rate = dt > 0 ? textLen / dt : 0;
62
+ this.timestamps.push(startTime);
63
+ this.speakingRate.push(rate);
64
+ this.speakIntegrals.push(newIntegral);
65
+ this.textBuffer = [];
66
+ }
67
+ this.textBuffer.push(text);
68
+ if (endTime !== void 0) {
69
+ this.addByAnnotation("", endTime, void 0);
70
+ }
71
+ }
72
+ /**
73
+ * Get accumulated speaking units up to the given timestamp.
74
+ */
75
+ accumulateTo(timestamp) {
76
+ if (this.timestamps.length === 0) {
77
+ return 0;
78
+ }
79
+ let idx = 0;
80
+ for (let i = 0; i < this.timestamps.length; i++) {
81
+ if (this.timestamps[i] <= timestamp) {
82
+ idx = i + 1;
83
+ } else {
84
+ break;
85
+ }
86
+ }
87
+ if (idx === 0) {
88
+ return 0;
89
+ }
90
+ let integralT = this.speakIntegrals[idx - 1];
91
+ const dt = timestamp - this.timestamps[idx - 1];
92
+ const rate = idx < this.speakingRate.length ? this.speakingRate[idx] : this.speakingRate[idx - 1];
93
+ integralT += rate * dt;
94
+ if (idx < this.timestamps.length) {
95
+ integralT = Math.min(integralT, this.speakIntegrals[idx]);
96
+ }
97
+ return integralT;
98
+ }
99
+ /** Get the last pushed timestamp. */
100
+ get pushedDuration() {
101
+ return this.timestamps.length > 0 ? this.timestamps[this.timestamps.length - 1] : 0;
102
+ }
103
+ }
31
104
  class SegmentSynchronizerImpl {
32
105
  constructor(options, nextInChain) {
33
106
  this.options = options;
@@ -42,7 +115,8 @@ class SegmentSynchronizerImpl {
42
115
  };
43
116
  this.audioData = {
44
117
  pushedDuration: 0,
45
- done: false
118
+ done: false,
119
+ annotatedRate: null
46
120
  };
47
121
  this.outputStream = new import_identity_transform.IdentityTransform();
48
122
  this.outputStreamWriter = this.outputStream.writable.getWriter();
@@ -73,6 +147,9 @@ class SegmentSynchronizerImpl {
73
147
  get textInputEnded() {
74
148
  return this.textData.done;
75
149
  }
150
+ get hasPendingText() {
151
+ return this.textData.pushedText.length > this.textData.forwardedText.length;
152
+ }
76
153
  get readable() {
77
154
  return this.outputStream.readable;
78
155
  }
@@ -100,8 +177,22 @@ class SegmentSynchronizerImpl {
100
177
  this.logger.warn("SegmentSynchronizerImpl.pushText called after close");
101
178
  return;
102
179
  }
103
- this.textData.sentenceStream.pushText(text);
104
- this.textData.pushedText += text;
180
+ let textStr;
181
+ let startTime;
182
+ let endTime;
183
+ if ((0, import_io.isTimedString)(text)) {
184
+ textStr = text.text;
185
+ startTime = text.startTime;
186
+ endTime = text.endTime;
187
+ if (!this.audioData.annotatedRate) {
188
+ this.audioData.annotatedRate = new SpeakingRateData();
189
+ }
190
+ this.audioData.annotatedRate.addByAnnotation(textStr, startTime, endTime);
191
+ } else {
192
+ textStr = text;
193
+ }
194
+ this.textData.sentenceStream.pushText(textStr);
195
+ this.textData.pushedText += textStr;
105
196
  }
106
197
  endTextInput() {
107
198
  if (this.closed) {
@@ -121,6 +212,9 @@ class SegmentSynchronizerImpl {
121
212
  { textDone: this.textData.done, audioDone: this.audioData.done },
122
213
  "SegmentSynchronizerImpl.markPlaybackFinished called before text/audio input is done"
123
214
  );
215
+ if (!interrupted) {
216
+ this.playbackCompleted = true;
217
+ }
124
218
  return;
125
219
  }
126
220
  if (!interrupted) {
@@ -140,7 +234,6 @@ class SegmentSynchronizerImpl {
140
234
  if (done) {
141
235
  break;
142
236
  }
143
- this.textData.forwardedText += text;
144
237
  await this.nextInChain.captureText(text);
145
238
  }
146
239
  reader.releaseLock();
@@ -171,16 +264,32 @@ class SegmentSynchronizerImpl {
171
264
  }
172
265
  const wordHphens = this.options.hyphenateWord(word).length;
173
266
  const elapsedSeconds = (Date.now() - this.startWallTime) / 1e3;
174
- const targetHyphens = elapsedSeconds * this.options.speed;
175
- const hyphensBehind = Math.max(0, targetHyphens - this.textData.forwardedHyphens);
176
- let delay2 = Math.max(0, wordHphens - hyphensBehind) / this.speed;
267
+ let dHyphens = 0;
268
+ const annotated = this.audioData.annotatedRate;
269
+ if (annotated && annotated.pushedDuration >= elapsedSeconds) {
270
+ const targetLen = Math.floor(annotated.accumulateTo(elapsedSeconds));
271
+ const forwardedLen = this.textData.forwardedText.length;
272
+ if (targetLen >= forwardedLen) {
273
+ const dText = this.textData.pushedText.slice(forwardedLen, targetLen);
274
+ dHyphens = this.calcHyphens(dText).length;
275
+ } else {
276
+ const dText = this.textData.pushedText.slice(targetLen, forwardedLen);
277
+ dHyphens = -this.calcHyphens(dText).length;
278
+ }
279
+ } else {
280
+ const targetHyphens = elapsedSeconds * this.options.speed;
281
+ dHyphens = Math.max(0, targetHyphens - this.textData.forwardedHyphens);
282
+ }
283
+ let delayTime = Math.max(0, wordHphens - dHyphens) / this.speed;
177
284
  if (this.playbackCompleted) {
178
- delay2 = 0;
285
+ delayTime = 0;
179
286
  }
180
- await this.sleepIfNotClosed(delay2 / 2);
181
- this.outputStreamWriter.write(sentence.slice(textCursor, endPos));
182
- await this.sleepIfNotClosed(delay2 / 2);
287
+ await this.sleepIfNotClosed(delayTime / 2);
288
+ const forwardedWord = sentence.slice(textCursor, endPos);
289
+ this.outputStreamWriter.write(forwardedWord);
290
+ await this.sleepIfNotClosed(delayTime / 2);
183
291
  this.textData.forwardedHyphens += wordHphens;
292
+ this.textData.forwardedText += forwardedWord;
184
293
  textCursor = endPos;
185
294
  }
186
295
  if (textCursor < sentence.length) {
@@ -189,6 +298,14 @@ class SegmentSynchronizerImpl {
189
298
  }
190
299
  }
191
300
  }
301
+ calcHyphens(text) {
302
+ const words = this.options.splitWords(text);
303
+ const hyphens = [];
304
+ for (const [word] of words) {
305
+ hyphens.push(...this.options.hyphenateWord(word));
306
+ }
307
+ return hyphens;
308
+ }
192
309
  async sleepIfNotClosed(sleepTimeSeconds) {
193
310
  if (this.closed) {
194
311
  return;
@@ -311,6 +428,10 @@ class SyncedAudioOutput extends import_io.AudioOutput {
311
428
  return;
312
429
  }
313
430
  if (!this.pushedDuration) {
431
+ if (this.synchronizer._impl.hasPendingText) {
432
+ this.synchronizer._impl.endAudioInput();
433
+ return;
434
+ }
314
435
  this.synchronizer.rotateSegment();
315
436
  return;
316
437
  }
@@ -345,8 +466,9 @@ class SyncedTextOutput extends import_io.TextOutput {
345
466
  logger = (0, import_log.log)();
346
467
  async captureText(text) {
347
468
  await this.synchronizer.barrier();
469
+ const textStr = (0, import_io.isTimedString)(text) ? text.text : text;
348
470
  if (!this.synchronizer.enabled) {
349
- await this.nextInChain.captureText(text);
471
+ await this.nextInChain.captureText(textStr);
350
472
  return;
351
473
  }
352
474
  this.capturing = true;
@@ -359,7 +481,8 @@ class SyncedTextOutput extends import_io.TextOutput {
359
481
  }
360
482
  this.synchronizer._impl.pushText(text);
361
483
  }
362
- flush() {
484
+ async flush() {
485
+ await this.synchronizer.barrier();
363
486
  if (!this.synchronizer.enabled) {
364
487
  this.nextInChain.flush();
365
488
  return;
@@ -373,6 +496,7 @@ class SyncedTextOutput extends import_io.TextOutput {
373
496
  }
374
497
  // Annotate the CommonJS export names for ESM import in node:
375
498
  0 && (module.exports = {
499
+ SpeakingRateData,
376
500
  TranscriptionSynchronizer,
377
501
  defaultTextSyncOptions
378
502
  });