@livekit/agents 1.0.40 → 1.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +20 -18
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +20 -18
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +5 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/inference/stt.cjs +2 -1
- package/dist/inference/stt.cjs.map +1 -1
- package/dist/inference/stt.d.ts.map +1 -1
- package/dist/inference/stt.js +2 -1
- package/dist/inference/stt.js.map +1 -1
- package/dist/llm/realtime.cjs.map +1 -1
- package/dist/llm/realtime.d.cts +5 -1
- package/dist/llm/realtime.d.ts +5 -1
- package/dist/llm/realtime.d.ts.map +1 -1
- package/dist/llm/realtime.js.map +1 -1
- package/dist/tts/stream_adapter.cjs +15 -1
- package/dist/tts/stream_adapter.cjs.map +1 -1
- package/dist/tts/stream_adapter.d.ts.map +1 -1
- package/dist/tts/stream_adapter.js +15 -1
- package/dist/tts/stream_adapter.js.map +1 -1
- package/dist/tts/tts.cjs.map +1 -1
- package/dist/tts/tts.d.cts +9 -1
- package/dist/tts/tts.d.ts +9 -1
- package/dist/tts/tts.d.ts.map +1 -1
- package/dist/tts/tts.js.map +1 -1
- package/dist/types.cjs +3 -0
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +4 -0
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -1
- package/dist/voice/agent.cjs +11 -1
- package/dist/voice/agent.cjs.map +1 -1
- package/dist/voice/agent.d.cts +7 -3
- package/dist/voice/agent.d.ts +7 -3
- package/dist/voice/agent.d.ts.map +1 -1
- package/dist/voice/agent.js +11 -1
- package/dist/voice/agent.js.map +1 -1
- package/dist/voice/agent_activity.cjs +30 -14
- package/dist/voice/agent_activity.cjs.map +1 -1
- package/dist/voice/agent_activity.d.cts +1 -0
- package/dist/voice/agent_activity.d.ts +1 -0
- package/dist/voice/agent_activity.d.ts.map +1 -1
- package/dist/voice/agent_activity.js +30 -14
- package/dist/voice/agent_activity.js.map +1 -1
- package/dist/voice/agent_session.cjs +5 -1
- package/dist/voice/agent_session.cjs.map +1 -1
- package/dist/voice/agent_session.d.cts +2 -0
- package/dist/voice/agent_session.d.ts +2 -0
- package/dist/voice/agent_session.d.ts.map +1 -1
- package/dist/voice/agent_session.js +5 -1
- package/dist/voice/agent_session.js.map +1 -1
- package/dist/voice/background_audio.cjs +2 -1
- package/dist/voice/background_audio.cjs.map +1 -1
- package/dist/voice/background_audio.d.cts +4 -2
- package/dist/voice/background_audio.d.ts +4 -2
- package/dist/voice/background_audio.d.ts.map +1 -1
- package/dist/voice/background_audio.js +2 -1
- package/dist/voice/background_audio.js.map +1 -1
- package/dist/voice/generation.cjs +58 -5
- package/dist/voice/generation.cjs.map +1 -1
- package/dist/voice/generation.d.cts +17 -3
- package/dist/voice/generation.d.ts +17 -3
- package/dist/voice/generation.d.ts.map +1 -1
- package/dist/voice/generation.js +63 -6
- package/dist/voice/generation.js.map +1 -1
- package/dist/voice/index.cjs.map +1 -1
- package/dist/voice/index.d.cts +1 -1
- package/dist/voice/index.d.ts +1 -1
- package/dist/voice/index.d.ts.map +1 -1
- package/dist/voice/index.js.map +1 -1
- package/dist/voice/io.cjs +22 -2
- package/dist/voice/io.cjs.map +1 -1
- package/dist/voice/io.d.cts +21 -5
- package/dist/voice/io.d.ts +21 -5
- package/dist/voice/io.d.ts.map +1 -1
- package/dist/voice/io.js +18 -1
- package/dist/voice/io.js.map +1 -1
- package/dist/voice/room_io/_output.cjs +3 -2
- package/dist/voice/room_io/_output.cjs.map +1 -1
- package/dist/voice/room_io/_output.d.cts +3 -3
- package/dist/voice/room_io/_output.d.ts +3 -3
- package/dist/voice/room_io/_output.d.ts.map +1 -1
- package/dist/voice/room_io/_output.js +4 -3
- package/dist/voice/room_io/_output.js.map +1 -1
- package/dist/voice/transcription/synchronizer.cjs +137 -13
- package/dist/voice/transcription/synchronizer.cjs.map +1 -1
- package/dist/voice/transcription/synchronizer.d.cts +34 -4
- package/dist/voice/transcription/synchronizer.d.ts +34 -4
- package/dist/voice/transcription/synchronizer.d.ts.map +1 -1
- package/dist/voice/transcription/synchronizer.js +141 -14
- package/dist/voice/transcription/synchronizer.js.map +1 -1
- package/dist/voice/transcription/synchronizer.test.cjs +151 -0
- package/dist/voice/transcription/synchronizer.test.cjs.map +1 -0
- package/dist/voice/transcription/synchronizer.test.js +150 -0
- package/dist/voice/transcription/synchronizer.test.js.map +1 -0
- package/package.json +1 -1
- package/src/cli.ts +20 -18
- package/src/index.ts +1 -0
- package/src/inference/stt.ts +9 -8
- package/src/llm/realtime.ts +5 -1
- package/src/tts/stream_adapter.ts +23 -1
- package/src/tts/tts.ts +10 -1
- package/src/types.ts +5 -0
- package/src/voice/agent.ts +19 -4
- package/src/voice/agent_activity.ts +38 -13
- package/src/voice/agent_session.ts +6 -0
- package/src/voice/background_audio.ts +6 -3
- package/src/voice/generation.ts +115 -10
- package/src/voice/index.ts +1 -1
- package/src/voice/io.ts +40 -5
- package/src/voice/room_io/_output.ts +6 -5
- package/src/voice/transcription/synchronizer.test.ts +206 -0
- package/src/voice/transcription/synchronizer.ts +202 -17
package/dist/voice/io.js
CHANGED
|
@@ -2,6 +2,20 @@ import { EventEmitter } from "node:events";
|
|
|
2
2
|
import { log } from "../log.js";
|
|
3
3
|
import { DeferredReadableStream } from "../stream/deferred_stream.js";
|
|
4
4
|
import { Future } from "../utils.js";
|
|
5
|
+
const TIMED_STRING_SYMBOL = Symbol.for("lk.TimedString");
|
|
6
|
+
function createTimedString(opts) {
|
|
7
|
+
return {
|
|
8
|
+
[TIMED_STRING_SYMBOL]: true,
|
|
9
|
+
text: opts.text,
|
|
10
|
+
startTime: opts.startTime,
|
|
11
|
+
endTime: opts.endTime,
|
|
12
|
+
confidence: opts.confidence,
|
|
13
|
+
startTimeOffset: opts.startTimeOffset
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function isTimedString(value) {
|
|
17
|
+
return typeof value === "object" && value !== null && TIMED_STRING_SYMBOL in value && value[TIMED_STRING_SYMBOL] === true;
|
|
18
|
+
}
|
|
5
19
|
class AudioInput {
|
|
6
20
|
deferredStream = new DeferredReadableStream();
|
|
7
21
|
get stream() {
|
|
@@ -249,6 +263,9 @@ export {
|
|
|
249
263
|
AgentOutput,
|
|
250
264
|
AudioInput,
|
|
251
265
|
AudioOutput,
|
|
252
|
-
|
|
266
|
+
TIMED_STRING_SYMBOL,
|
|
267
|
+
TextOutput,
|
|
268
|
+
createTimedString,
|
|
269
|
+
isTimedString
|
|
253
270
|
};
|
|
254
271
|
//# sourceMappingURL=io.js.map
|
package/dist/voice/io.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/voice/io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport type { ChatContext } from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport type { ToolContext } from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport type { SpeechEvent } from '../stt/stt.js';\nimport { Future } from '../utils.js';\nimport type { ModelSettings } from './agent.js';\n\nexport type STTNode = (\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<SpeechEvent | string> | null>;\n\nexport type LLMNode = (\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<ChatChunk | string> | null>;\n\nexport type TTSNode = (\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<AudioFrame> | null>;\n\n/**\n *A string with optional start and end timestamps for word-level alignment.\n */\nexport interface TimedString {\n text: string;\n startTime?: number; // seconds\n endTime?: number; // seconds\n confidence?: number;\n startTimeOffset?: number;\n}\n\nexport interface AudioOutputCapabilities {\n /** Whether this output supports pause/resume functionality */\n pause: boolean;\n}\n\nexport abstract class AudioInput {\n protected deferredStream: DeferredReadableStream<AudioFrame> =\n new DeferredReadableStream<AudioFrame>();\n\n get stream(): ReadableStream<AudioFrame> {\n return this.deferredStream.stream;\n }\n\n onAttached(): void {}\n\n onDetached(): void {}\n}\n\nexport abstract class AudioOutput extends EventEmitter {\n static readonly EVENT_PLAYBACK_STARTED = 'playbackStarted';\n static readonly EVENT_PLAYBACK_FINISHED = 'playbackFinished';\n\n private playbackFinishedFuture: Future<void> = new Future();\n private _capturing: boolean = false;\n private playbackFinishedCount: number = 0;\n private playbackSegmentsCount: number = 0;\n private lastPlaybackEvent: PlaybackFinishedEvent = {\n playbackPosition: 0,\n interrupted: false,\n };\n protected logger = log();\n protected readonly capabilities: AudioOutputCapabilities;\n\n constructor(\n public sampleRate?: number,\n protected readonly nextInChain?: AudioOutput,\n capabilities: AudioOutputCapabilities = { pause: false },\n ) {\n super();\n this.capabilities = capabilities;\n\n if (this.nextInChain) {\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, (ev: PlaybackStartedEvent) =>\n this.onPlaybackStarted(ev.createdAt),\n );\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) =>\n this.onPlaybackFinished(ev),\n );\n }\n }\n\n /**\n * Whether this output and all outputs in the chain support pause/resume.\n */\n get canPause(): boolean {\n return this.capabilities.pause && (this.nextInChain?.canPause ?? true);\n }\n\n /**\n * Capture an audio frame for playback, frames can be pushed faster than real-time\n */\n async captureFrame(_frame: AudioFrame): Promise<void> {\n if (!this._capturing) {\n this._capturing = true;\n this.playbackSegmentsCount++;\n }\n }\n\n /**\n * Wait for the past audio segments to finish playing out.\n *\n * @returns The event that was emitted when the audio finished playing out (only the last segment information)\n */\n async waitForPlayout(): Promise<PlaybackFinishedEvent> {\n const target = this.playbackSegmentsCount;\n\n while (this.playbackFinishedCount < target) {\n await this.playbackFinishedFuture.await;\n this.playbackFinishedFuture = new Future();\n }\n\n return this.lastPlaybackEvent;\n }\n\n /**\n * Called when playback actually starts (first frame is sent to output).\n * Developers building audio sinks should call this when the first frame is captured.\n */\n onPlaybackStarted(createdAt: number): void {\n this.emit(AudioOutput.EVENT_PLAYBACK_STARTED, { createdAt } as PlaybackStartedEvent);\n }\n\n /**\n * Developers building audio sinks must call this method when a playback/segment is finished.\n * Segments are segmented by calls to flush() or clearBuffer()\n */\n onPlaybackFinished(options: PlaybackFinishedEvent) {\n if (this.playbackFinishedCount >= this.playbackSegmentsCount) {\n this.logger.warn('playback_finished called more times than playback segments were captured');\n return;\n }\n\n this.lastPlaybackEvent = options;\n this.playbackFinishedCount++;\n this.playbackFinishedFuture.resolve();\n this.emit(AudioOutput.EVENT_PLAYBACK_FINISHED, options);\n }\n\n flush(): void {\n this._capturing = false;\n }\n\n /**\n * Clear the buffer, stopping playback immediately\n */\n abstract clearBuffer(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n\n /**\n * Pause the audio playback\n */\n pause(): void {\n if (this.nextInChain) {\n this.nextInChain.pause();\n }\n }\n\n /**\n * Resume the audio playback\n */\n resume(): void {\n if (this.nextInChain) {\n this.nextInChain.resume();\n }\n }\n}\n\nexport interface PlaybackFinishedEvent {\n /** How much of the audio was played back, in seconds */\n playbackPosition: number;\n /** True if playback was interrupted (clearBuffer() was called) */\n interrupted: boolean;\n /**\n * Transcript synced with playback; may be partial if the audio was interrupted.\n * When undefined, the transcript is not synchronized with the playback.\n */\n synchronizedTranscript?: string;\n}\n\nexport interface PlaybackStartedEvent {\n /** The timestamp (Date.now()) when the playback started */\n createdAt: number;\n}\n\nexport abstract class TextOutput {\n constructor(protected readonly nextInChain?: TextOutput) {}\n\n /**\n * Capture a text segment (Used by the output of LLM nodes)\n */\n abstract captureText(text: string): Promise<void>;\n\n /**\n * Mark the current text segment as complete (e.g LLM generation is complete)\n */\n abstract flush(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n}\n\nexport class AgentInput {\n private _audioStream: AudioInput | null = null;\n // enabled by default\n private _audioEnabled: boolean = true;\n\n constructor(private readonly audioChanged: () => void) {}\n\n setAudioEnabled(enable: boolean): void {\n if (enable === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enable;\n\n if (!this._audioStream) {\n return;\n }\n\n if (enable) {\n this._audioStream.onAttached();\n } else {\n this._audioStream.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get audio(): AudioInput | null {\n return this._audioStream;\n }\n\n set audio(stream: AudioInput | null) {\n this._audioStream = stream;\n this.audioChanged();\n }\n}\n\nexport class AgentOutput {\n private _audioSink: AudioOutput | null = null;\n private _transcriptionSink: TextOutput | null = null;\n private _audioEnabled: boolean = true;\n private _transcriptionEnabled: boolean = true;\n\n constructor(\n private readonly audioChanged: () => void,\n private readonly transcriptionChanged: () => void,\n ) {}\n\n setAudioEnabled(enabled: boolean): void {\n if (enabled === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enabled;\n\n if (!this._audioSink) {\n return;\n }\n\n if (enabled) {\n this._audioSink.onAttached();\n } else {\n this._audioSink.onDetached();\n }\n }\n\n setTranscriptionEnabled(enabled: boolean): void {\n if (enabled === this._transcriptionEnabled) {\n return;\n }\n\n this._transcriptionEnabled = enabled;\n\n if (!this._transcriptionSink) {\n return;\n }\n\n if (enabled) {\n this._transcriptionSink.onAttached();\n } else {\n this._transcriptionSink.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get transcriptionEnabled(): boolean {\n return this._transcriptionEnabled;\n }\n\n get audio(): AudioOutput | null {\n return this._audioSink;\n }\n\n set audio(sink: AudioOutput | null) {\n if (sink === this._audioSink) {\n return;\n }\n\n if (this._audioSink) {\n this._audioSink.onDetached();\n }\n\n this._audioSink = sink;\n this.audioChanged();\n\n if (this._audioSink) {\n this._audioSink.onAttached();\n }\n }\n\n get transcription(): TextOutput | null {\n return this._transcriptionSink;\n }\n\n set transcription(sink: TextOutput | null) {\n if (sink === this._transcriptionSink) {\n return;\n }\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onDetached();\n }\n\n this._transcriptionSink = sink;\n this.transcriptionChanged();\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onAttached();\n }\n }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAK7B,SAAS,WAAW;AACpB,SAAS,8BAA8B;AAEvC,SAAS,cAAc;AAmChB,MAAe,WAAW;AAAA,EACrB,iBACR,IAAI,uBAAmC;AAAA,EAEzC,IAAI,SAAqC;AACvC,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,aAAmB;AAAA,EAAC;AAAA,EAEpB,aAAmB;AAAA,EAAC;AACtB;AAEO,MAAe,oBAAoB,aAAa;AAAA,EAerD,YACS,YACY,aACnB,eAAwC,EAAE,OAAO,MAAM,GACvD;AACA,UAAM;AAJC;AACY;AAInB,SAAK,eAAe;AAEpB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAwB,CAAC,OACvD,KAAK,kBAAkB,GAAG,SAAS;AAAA,MACrC;AACA,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAyB,CAAC,OACxD,KAAK,mBAAmB,EAAE;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAAA,EA9BA,OAAgB,yBAAyB;AAAA,EACzC,OAAgB,0BAA0B;AAAA,EAElC,yBAAuC,IAAI,OAAO;AAAA,EAClD,aAAsB;AAAA,EACtB,wBAAgC;AAAA,EAChC,wBAAgC;AAAA,EAChC,oBAA2C;AAAA,IACjD,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf;AAAA,EACU,SAAS,IAAI;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA,EAuBnB,IAAI,WAAoB;AAhG1B;AAiGI,WAAO,KAAK,aAAa,YAAU,UAAK,gBAAL,mBAAkB,aAAY;AAAA,EACnE;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aAAa,QAAmC;AACpD,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa;AAClB,WAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,iBAAiD;AACrD,UAAM,SAAS,KAAK;AAEpB,WAAO,KAAK,wBAAwB,QAAQ;AAC1C,YAAM,KAAK,uBAAuB;AAClC,WAAK,yBAAyB,IAAI,OAAO;AAAA,IAC3C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAkB,WAAyB;AACzC,SAAK,KAAK,YAAY,wBAAwB,EAAE,UAAU,CAAyB;AAAA,EACrF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,mBAAmB,SAAgC;AACjD,QAAI,KAAK,yBAAyB,KAAK,uBAAuB;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAC3F;AAAA,IACF;AAEA,SAAK,oBAAoB;AACzB,SAAK;AACL,SAAK,uBAAuB,QAAQ;AACpC,SAAK,KAAK,YAAY,yBAAyB,OAAO;AAAA,EACxD;AAAA,EAEA,QAAc;AACZ,SAAK,aAAa;AAAA,EACpB;AAAA,EAOA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,MAAM;AAAA,IACzB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,SAAe;AACb,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,OAAO;AAAA,IAC1B;AAAA,EACF;AACF;AAmBO,MAAe,WAAW;AAAA,EAC/B,YAA+B,aAA0B;AAA1B;AAAA,EAA2B;AAAA,EAY1D,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AACF;AAEO,MAAM,WAAW;AAAA,EAKtB,YAA6B,cAA0B;AAA1B;AAAA,EAA2B;AAAA,EAJhD,eAAkC;AAAA;AAAA,EAElC,gBAAyB;AAAA,EAIjC,gBAAgB,QAAuB;AACrC,QAAI,WAAW,KAAK,eAAe;AACjC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,QAAI,QAAQ;AACV,WAAK,aAAa,WAAW;AAAA,IAC/B,OAAO;AACL,WAAK,aAAa,WAAW;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,QAA2B;AACnC,SAAK,eAAe;AACpB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,YAAY;AAAA,EAMvB,YACmB,cACA,sBACjB;AAFiB;AACA;AAAA,EAChB;AAAA,EARK,aAAiC;AAAA,EACjC,qBAAwC;AAAA,EACxC,gBAAyB;AAAA,EACzB,wBAAiC;AAAA,EAOzC,gBAAgB,SAAwB;AACtC,QAAI,YAAY,KAAK,eAAe;AAClC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,WAAW,WAAW;AAAA,IAC7B,OAAO;AACL,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,wBAAwB,SAAwB;AAC9C,QAAI,YAAY,KAAK,uBAAuB;AAC1C;AAAA,IACF;AAEA,SAAK,wBAAwB;AAE7B,QAAI,CAAC,KAAK,oBAAoB;AAC5B;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,mBAAmB,WAAW;AAAA,IACrC,OAAO;AACL,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,uBAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,MAA0B;AAClC,QAAI,SAAS,KAAK,YAAY;AAC5B;AAAA,IACF;AAEA,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAEA,SAAK,aAAa;AAClB,SAAK,aAAa;AAElB,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,IAAI,gBAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAc,MAAyB;AACzC,QAAI,SAAS,KAAK,oBAAoB;AACpC;AAAA,IACF;AAEA,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAEA,SAAK,qBAAqB;AAC1B,SAAK,qBAAqB;AAE1B,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/voice/io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport type { ChatContext } from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport type { ToolContext } from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport type { SpeechEvent } from '../stt/stt.js';\nimport { Future } from '../utils.js';\nimport type { ModelSettings } from './agent.js';\n\nexport type STTNode = (\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<SpeechEvent | string> | null>;\n\nexport type LLMNode = (\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<ChatChunk | string> | null>;\n\nexport type TTSNode = (\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n) => Promise<ReadableStream<AudioFrame> | null>;\n\n/**\n * Symbol used to identify TimedString objects.\n */\nexport const TIMED_STRING_SYMBOL = Symbol.for('lk.TimedString');\n\n/**\n * A string with optional start and end timestamps for word-level alignment.\n */\nexport interface TimedString {\n readonly [TIMED_STRING_SYMBOL]: true;\n text: string;\n startTime?: number; // seconds\n endTime?: number; // seconds\n confidence?: number;\n startTimeOffset?: number;\n}\n\n/**\n * Factory function to create a TimedString object.\n */\nexport function createTimedString(opts: {\n text: string;\n startTime?: number;\n endTime?: number;\n confidence?: number;\n startTimeOffset?: number;\n}): TimedString {\n return {\n [TIMED_STRING_SYMBOL]: true,\n text: opts.text,\n startTime: opts.startTime,\n endTime: opts.endTime,\n confidence: opts.confidence,\n startTimeOffset: opts.startTimeOffset,\n };\n}\n\n/**\n * Type guard to check if a value is a TimedString.\n */\nexport function isTimedString(value: unknown): value is TimedString {\n return (\n typeof value === 'object' &&\n value !== null &&\n TIMED_STRING_SYMBOL in value &&\n (value as TimedString)[TIMED_STRING_SYMBOL] === true\n );\n}\n\nexport interface AudioOutputCapabilities {\n /** Whether this output supports pause/resume functionality */\n pause: boolean;\n}\n\nexport abstract class AudioInput {\n protected deferredStream: DeferredReadableStream<AudioFrame> =\n new DeferredReadableStream<AudioFrame>();\n\n get stream(): ReadableStream<AudioFrame> {\n return this.deferredStream.stream;\n }\n\n onAttached(): void {}\n\n onDetached(): void {}\n}\n\nexport abstract class AudioOutput extends EventEmitter {\n static readonly EVENT_PLAYBACK_STARTED = 'playbackStarted';\n static readonly EVENT_PLAYBACK_FINISHED = 'playbackFinished';\n\n private playbackFinishedFuture: Future<void> = new Future();\n private _capturing: boolean = false;\n private playbackFinishedCount: number = 0;\n private playbackSegmentsCount: number = 0;\n private lastPlaybackEvent: PlaybackFinishedEvent = {\n playbackPosition: 0,\n interrupted: false,\n };\n protected logger = log();\n protected readonly capabilities: AudioOutputCapabilities;\n\n constructor(\n public sampleRate?: number,\n protected readonly nextInChain?: AudioOutput,\n capabilities: AudioOutputCapabilities = { pause: false },\n ) {\n super();\n this.capabilities = capabilities;\n\n if (this.nextInChain) {\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, (ev: PlaybackStartedEvent) =>\n this.onPlaybackStarted(ev.createdAt),\n );\n this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) =>\n this.onPlaybackFinished(ev),\n );\n }\n }\n\n /**\n * Whether this output and all outputs in the chain support pause/resume.\n */\n get canPause(): boolean {\n return this.capabilities.pause && (this.nextInChain?.canPause ?? true);\n }\n\n /**\n * Capture an audio frame for playback, frames can be pushed faster than real-time\n */\n async captureFrame(_frame: AudioFrame): Promise<void> {\n if (!this._capturing) {\n this._capturing = true;\n this.playbackSegmentsCount++;\n }\n }\n\n /**\n * Wait for the past audio segments to finish playing out.\n *\n * @returns The event that was emitted when the audio finished playing out (only the last segment information)\n */\n async waitForPlayout(): Promise<PlaybackFinishedEvent> {\n const target = this.playbackSegmentsCount;\n\n while (this.playbackFinishedCount < target) {\n await this.playbackFinishedFuture.await;\n this.playbackFinishedFuture = new Future();\n }\n\n return this.lastPlaybackEvent;\n }\n\n /**\n * Called when playback actually starts (first frame is sent to output).\n * Developers building audio sinks should call this when the first frame is captured.\n */\n onPlaybackStarted(createdAt: number): void {\n this.emit(AudioOutput.EVENT_PLAYBACK_STARTED, { createdAt } as PlaybackStartedEvent);\n }\n\n /**\n * Developers building audio sinks must call this method when a playback/segment is finished.\n * Segments are segmented by calls to flush() or clearBuffer()\n */\n onPlaybackFinished(options: PlaybackFinishedEvent) {\n if (this.playbackFinishedCount >= this.playbackSegmentsCount) {\n this.logger.warn('playback_finished called more times than playback segments were captured');\n return;\n }\n\n this.lastPlaybackEvent = options;\n this.playbackFinishedCount++;\n this.playbackFinishedFuture.resolve();\n this.emit(AudioOutput.EVENT_PLAYBACK_FINISHED, options);\n }\n\n flush(): void {\n this._capturing = false;\n }\n\n /**\n * Clear the buffer, stopping playback immediately\n */\n abstract clearBuffer(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n\n /**\n * Pause the audio playback\n */\n pause(): void {\n if (this.nextInChain) {\n this.nextInChain.pause();\n }\n }\n\n /**\n * Resume the audio playback\n */\n resume(): void {\n if (this.nextInChain) {\n this.nextInChain.resume();\n }\n }\n}\n\nexport interface PlaybackFinishedEvent {\n /** How much of the audio was played back, in seconds */\n playbackPosition: number;\n /** True if playback was interrupted (clearBuffer() was called) */\n interrupted: boolean;\n /**\n * Transcript synced with playback; may be partial if the audio was interrupted.\n * When undefined, the transcript is not synchronized with the playback.\n */\n synchronizedTranscript?: string;\n}\n\nexport interface PlaybackStartedEvent {\n /** The timestamp (Date.now()) when the playback started */\n createdAt: number;\n}\n\nexport abstract class TextOutput {\n constructor(protected readonly nextInChain?: TextOutput) {}\n\n abstract captureText(text: string | TimedString): Promise<void>;\n\n /**\n * Mark the current text segment as complete (e.g LLM generation is complete)\n */\n abstract flush(): void;\n\n onAttached(): void {\n if (this.nextInChain) {\n this.nextInChain.onAttached();\n }\n }\n\n onDetached(): void {\n if (this.nextInChain) {\n this.nextInChain.onDetached();\n }\n }\n}\n\nexport class AgentInput {\n private _audioStream: AudioInput | null = null;\n // enabled by default\n private _audioEnabled: boolean = true;\n\n constructor(private readonly audioChanged: () => void) {}\n\n setAudioEnabled(enable: boolean): void {\n if (enable === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enable;\n\n if (!this._audioStream) {\n return;\n }\n\n if (enable) {\n this._audioStream.onAttached();\n } else {\n this._audioStream.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get audio(): AudioInput | null {\n return this._audioStream;\n }\n\n set audio(stream: AudioInput | null) {\n this._audioStream = stream;\n this.audioChanged();\n }\n}\n\nexport class AgentOutput {\n private _audioSink: AudioOutput | null = null;\n private _transcriptionSink: TextOutput | null = null;\n private _audioEnabled: boolean = true;\n private _transcriptionEnabled: boolean = true;\n\n constructor(\n private readonly audioChanged: () => void,\n private readonly transcriptionChanged: () => void,\n ) {}\n\n setAudioEnabled(enabled: boolean): void {\n if (enabled === this._audioEnabled) {\n return;\n }\n\n this._audioEnabled = enabled;\n\n if (!this._audioSink) {\n return;\n }\n\n if (enabled) {\n this._audioSink.onAttached();\n } else {\n this._audioSink.onDetached();\n }\n }\n\n setTranscriptionEnabled(enabled: boolean): void {\n if (enabled === this._transcriptionEnabled) {\n return;\n }\n\n this._transcriptionEnabled = enabled;\n\n if (!this._transcriptionSink) {\n return;\n }\n\n if (enabled) {\n this._transcriptionSink.onAttached();\n } else {\n this._transcriptionSink.onDetached();\n }\n }\n\n get audioEnabled(): boolean {\n return this._audioEnabled;\n }\n\n get transcriptionEnabled(): boolean {\n return this._transcriptionEnabled;\n }\n\n get audio(): AudioOutput | null {\n return this._audioSink;\n }\n\n set audio(sink: AudioOutput | null) {\n if (sink === this._audioSink) {\n return;\n }\n\n if (this._audioSink) {\n this._audioSink.onDetached();\n }\n\n this._audioSink = sink;\n this.audioChanged();\n\n if (this._audioSink) {\n this._audioSink.onAttached();\n }\n }\n\n get transcription(): TextOutput | null {\n return this._transcriptionSink;\n }\n\n set transcription(sink: TextOutput | null) {\n if (sink === this._transcriptionSink) {\n return;\n }\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onDetached();\n }\n\n this._transcriptionSink = sink;\n this.transcriptionChanged();\n\n if (this._transcriptionSink) {\n this._transcriptionSink.onAttached();\n }\n }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAK7B,SAAS,WAAW;AACpB,SAAS,8BAA8B;AAEvC,SAAS,cAAc;AAsBhB,MAAM,sBAAsB,OAAO,IAAI,gBAAgB;AAiBvD,SAAS,kBAAkB,MAMlB;AACd,SAAO;AAAA,IACL,CAAC,mBAAmB,GAAG;AAAA,IACvB,MAAM,KAAK;AAAA,IACX,WAAW,KAAK;AAAA,IAChB,SAAS,KAAK;AAAA,IACd,YAAY,KAAK;AAAA,IACjB,iBAAiB,KAAK;AAAA,EACxB;AACF;AAKO,SAAS,cAAc,OAAsC;AAClE,SACE,OAAO,UAAU,YACjB,UAAU,QACV,uBAAuB,SACtB,MAAsB,mBAAmB,MAAM;AAEpD;AAOO,MAAe,WAAW;AAAA,EACrB,iBACR,IAAI,uBAAmC;AAAA,EAEzC,IAAI,SAAqC;AACvC,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,aAAmB;AAAA,EAAC;AAAA,EAEpB,aAAmB;AAAA,EAAC;AACtB;AAEO,MAAe,oBAAoB,aAAa;AAAA,EAerD,YACS,YACY,aACnB,eAAwC,EAAE,OAAO,MAAM,GACvD;AACA,UAAM;AAJC;AACY;AAInB,SAAK,eAAe;AAEpB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAwB,CAAC,OACvD,KAAK,kBAAkB,GAAG,SAAS;AAAA,MACrC;AACA,WAAK,YAAY;AAAA,QAAG,YAAY;AAAA,QAAyB,CAAC,OACxD,KAAK,mBAAmB,EAAE;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAAA,EA9BA,OAAgB,yBAAyB;AAAA,EACzC,OAAgB,0BAA0B;AAAA,EAElC,yBAAuC,IAAI,OAAO;AAAA,EAClD,aAAsB;AAAA,EACtB,wBAAgC;AAAA,EAChC,wBAAgC;AAAA,EAChC,oBAA2C;AAAA,IACjD,kBAAkB;AAAA,IAClB,aAAa;AAAA,EACf;AAAA,EACU,SAAS,IAAI;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA,EAuBnB,IAAI,WAAoB;AAtI1B;AAuII,WAAO,KAAK,aAAa,YAAU,UAAK,gBAAL,mBAAkB,aAAY;AAAA,EACnE;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aAAa,QAAmC;AACpD,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa;AAClB,WAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,iBAAiD;AACrD,UAAM,SAAS,KAAK;AAEpB,WAAO,KAAK,wBAAwB,QAAQ;AAC1C,YAAM,KAAK,uBAAuB;AAClC,WAAK,yBAAyB,IAAI,OAAO;AAAA,IAC3C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAkB,WAAyB;AACzC,SAAK,KAAK,YAAY,wBAAwB,EAAE,UAAU,CAAyB;AAAA,EACrF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,mBAAmB,SAAgC;AACjD,QAAI,KAAK,yBAAyB,KAAK,uBAAuB;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAC3F;AAAA,IACF;AAEA,SAAK,oBAAoB;AACzB,SAAK;AACL,SAAK,uBAAuB,QAAQ;AACpC,SAAK,KAAK,YAAY,yBAAyB,OAAO;AAAA,EACxD;AAAA,EAEA,QAAc;AACZ,SAAK,aAAa;AAAA,EACpB;AAAA,EAOA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,MAAM;AAAA,IACzB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,SAAe;AACb,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,OAAO;AAAA,IAC1B;AAAA,EACF;AACF;AAmBO,MAAe,WAAW;AAAA,EAC/B,YAA+B,aAA0B;AAA1B;AAAA,EAA2B;AAAA,EAS1D,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,aAAmB;AACjB,QAAI,KAAK,aAAa;AACpB,WAAK,YAAY,WAAW;AAAA,IAC9B;AAAA,EACF;AACF;AAEO,MAAM,WAAW;AAAA,EAKtB,YAA6B,cAA0B;AAA1B;AAAA,EAA2B;AAAA,EAJhD,eAAkC;AAAA;AAAA,EAElC,gBAAyB;AAAA,EAIjC,gBAAgB,QAAuB;AACrC,QAAI,WAAW,KAAK,eAAe;AACjC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,QAAI,QAAQ;AACV,WAAK,aAAa,WAAW;AAAA,IAC/B,OAAO;AACL,WAAK,aAAa,WAAW;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,QAA2B;AACnC,SAAK,eAAe;AACpB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,YAAY;AAAA,EAMvB,YACmB,cACA,sBACjB;AAFiB;AACA;AAAA,EAChB;AAAA,EARK,aAAiC;AAAA,EACjC,qBAAwC;AAAA,EACxC,gBAAyB;AAAA,EACzB,wBAAiC;AAAA,EAOzC,gBAAgB,SAAwB;AACtC,QAAI,YAAY,KAAK,eAAe;AAClC;AAAA,IACF;AAEA,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,WAAW,WAAW;AAAA,IAC7B,OAAO;AACL,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,wBAAwB,SAAwB;AAC9C,QAAI,YAAY,KAAK,uBAAuB;AAC1C;AAAA,IACF;AAEA,SAAK,wBAAwB;AAE7B,QAAI,CAAC,KAAK,oBAAoB;AAC5B;AAAA,IACF;AAEA,QAAI,SAAS;AACX,WAAK,mBAAmB,WAAW;AAAA,IACrC,OAAO;AACL,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,uBAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAA4B;AAC9B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAM,MAA0B;AAClC,QAAI,SAAS,KAAK,YAAY;AAC5B;AAAA,IACF;AAEA,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAEA,SAAK,aAAa;AAClB,SAAK,aAAa;AAElB,QAAI,KAAK,YAAY;AACnB,WAAK,WAAW,WAAW;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,IAAI,gBAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAc,MAAyB;AACzC,QAAI,SAAS,KAAK,oBAAoB;AACpC;AAAA,IACF;AAEA,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAEA,SAAK,qBAAqB;AAC1B,SAAK,qBAAqB;AAE1B,QAAI,KAAK,oBAAoB;AAC3B,WAAK,mBAAmB,WAAW;AAAA,IACrC;AAAA,EACF;AACF;","names":[]}
|
|
@@ -88,8 +88,9 @@ class BaseParticipantTranscriptionOutput extends import_io.TextOutput {
|
|
|
88
88
|
if (!this.participantIdentity) {
|
|
89
89
|
return;
|
|
90
90
|
}
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
const textStr = (0, import_io.isTimedString)(text) ? text.text : text;
|
|
92
|
+
this.latestText = textStr;
|
|
93
|
+
await this.handleCaptureText(textStr);
|
|
93
94
|
}
|
|
94
95
|
flush() {
|
|
95
96
|
if (!this.participantIdentity || !this.capturing) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string) {\n if (!this.participantIdentity) {\n return;\n }\n\n this.latestText = text;\n await this.handleCaptureText(text);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAYO;AACP,uBAKO;AACP,iBAAoB;AACpB,mBAAwC;AACxC,gBAAwC;AACxC,2BAAsC;AAEtC,MAAe,2CAA2C,qBAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,aAAS,gBAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,0BAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,0BAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,cAAU,4CAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,eAAO,wBAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,IAAI;AAAA,EACnC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,8CAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,iDAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,mDAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,8CAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,iDAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAvQ7F;AAwQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,qBAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,sBAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,oBAAO;AAAA,EACzC,oBAAkC,IAAI,oBAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,4BAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,oBAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,oBAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAxalD;AAyaI,UAAM,QAAQ,gCAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oCAAoB,EAAE,QAAQ,4BAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string | TimedString) {\n if (!this.participantIdentity) {\n return;\n }\n\n const textStr = isTimedString(text) ? text.text : text;\n this.latestText = textStr;\n await this.handleCaptureText(textStr);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string | TimedString) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAYO;AACP,uBAKO;AACP,iBAAoB;AACpB,mBAAwC;AACxC,gBAAyE;AACzE,2BAAsC;AAEtC,MAAe,2CAA2C,qBAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,aAAS,gBAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,0BAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,0BAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,cAAU,4CAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,4BAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,eAAO,wBAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,UAAM,cAAU,yBAAc,IAAI,IAAI,KAAK,OAAO;AAClD,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,OAAO;AAAA,EACtC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,8CAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,iDAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,mDAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,8CAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,iDAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAxQ7F;AAyQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,qBAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,sBAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,oBAAO;AAAA,EACzC,oBAAkC,IAAI,oBAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,4BAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,oBAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,oBAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAzalD;AA0aI,UAAM,QAAQ,gCAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oCAAoB,EAAE,QAAQ,4BAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { RemoteParticipant } from '@livekit/rtc-node';
|
|
2
2
|
import { type AudioFrame, type LocalTrackPublication, type Participant, type RemoteTrackPublication, type Room, TrackPublishOptions } from '@livekit/rtc-node';
|
|
3
|
-
import { AudioOutput, TextOutput } from '../io.js';
|
|
3
|
+
import { AudioOutput, TextOutput, type TimedString } from '../io.js';
|
|
4
4
|
declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
|
|
5
5
|
protected room: Room;
|
|
6
6
|
protected isDeltaStream: boolean;
|
|
@@ -16,7 +16,7 @@ declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
|
|
|
16
16
|
protected onLocalTrackPublished: (track: LocalTrackPublication) => void;
|
|
17
17
|
protected generateCurrentId(): string;
|
|
18
18
|
protected resetState(): void;
|
|
19
|
-
captureText(text: string): Promise<void>;
|
|
19
|
+
captureText(text: string | TimedString): Promise<void>;
|
|
20
20
|
flush(): void;
|
|
21
21
|
protected abstract handleCaptureText(text: string): Promise<void>;
|
|
22
22
|
protected abstract handleFlush(): void;
|
|
@@ -41,7 +41,7 @@ export declare class ParalellTextOutput extends TextOutput {
|
|
|
41
41
|
/** @internal */
|
|
42
42
|
_sinks: TextOutput[];
|
|
43
43
|
constructor(sinks: TextOutput[], nextInChain?: TextOutput);
|
|
44
|
-
captureText(text: string): Promise<void>;
|
|
44
|
+
captureText(text: string | TimedString): Promise<void>;
|
|
45
45
|
flush(): void;
|
|
46
46
|
}
|
|
47
47
|
export interface AudioOutputOptions {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { RemoteParticipant } from '@livekit/rtc-node';
|
|
2
2
|
import { type AudioFrame, type LocalTrackPublication, type Participant, type RemoteTrackPublication, type Room, TrackPublishOptions } from '@livekit/rtc-node';
|
|
3
|
-
import { AudioOutput, TextOutput } from '../io.js';
|
|
3
|
+
import { AudioOutput, TextOutput, type TimedString } from '../io.js';
|
|
4
4
|
declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
|
|
5
5
|
protected room: Room;
|
|
6
6
|
protected isDeltaStream: boolean;
|
|
@@ -16,7 +16,7 @@ declare abstract class BaseParticipantTranscriptionOutput extends TextOutput {
|
|
|
16
16
|
protected onLocalTrackPublished: (track: LocalTrackPublication) => void;
|
|
17
17
|
protected generateCurrentId(): string;
|
|
18
18
|
protected resetState(): void;
|
|
19
|
-
captureText(text: string): Promise<void>;
|
|
19
|
+
captureText(text: string | TimedString): Promise<void>;
|
|
20
20
|
flush(): void;
|
|
21
21
|
protected abstract handleCaptureText(text: string): Promise<void>;
|
|
22
22
|
protected abstract handleFlush(): void;
|
|
@@ -41,7 +41,7 @@ export declare class ParalellTextOutput extends TextOutput {
|
|
|
41
41
|
/** @internal */
|
|
42
42
|
_sinks: TextOutput[];
|
|
43
43
|
constructor(sinks: TextOutput[], nextInChain?: TextOutput);
|
|
44
|
-
captureText(text: string): Promise<void>;
|
|
44
|
+
captureText(text: string | TimedString): Promise<void>;
|
|
45
45
|
flush(): void;
|
|
46
46
|
}
|
|
47
47
|
export interface AudioOutputOptions {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"_output.d.ts","sourceRoot":"","sources":["../../../src/voice/room_io/_output.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,sBAAsB,EAC3B,KAAK,IAAI,EAGT,mBAAmB,EAEpB,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"_output.d.ts","sourceRoot":"","sources":["../../../src/voice/room_io/_output.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,sBAAsB,EAC3B,KAAK,IAAI,EAGT,mBAAmB,EAEpB,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,KAAK,WAAW,EAAiB,MAAM,UAAU,CAAC;AAGpF,uBAAe,kCAAmC,SAAQ,UAAU;IAClE,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC;IACrB,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAQ;IACpD,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,SAAS,EAAE,OAAO,CAAS;IACrC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAM;IAClC,SAAS,CAAC,SAAS,EAAE,MAAM,CAA4B;IACvD,SAAS,CAAC,MAAM,wBAAS;gBAEb,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAWxF,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAqBvD,SAAS,CAAC,gBAAgB,UAAW,sBAAsB,eAAe,iBAAiB,UAUzF;IAEF,SAAS,CAAC,qBAAqB,UAAW,qBAAqB,UAU7D;IAEF,SAAS,CAAC,iBAAiB,IAAI,MAAM;IAIrC,SAAS,CAAC,UAAU;IAMd,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW;IAU5C,KAAK;IASL,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IACjE,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI;CACvC;AAED,qBAAa,8BAA+B,SAAQ,kCAAkC;IACpF,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,SAAS,CAA2B;cAE5B,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B9D,SAAS,CAAC,WAAW;YAMP,gBAAgB;YA0BhB,aAAa;CAkC5B;AAED,qBAAa,oCAAqC,SAAQ,kCAAkC;IAC1F,OAAO,CAAC,UAAU,CAAc;IAChC,OAAO,CAAC,SAAS,CAA8B;cAE/B,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB9D,SAAS,CAAC,WAAW;IASf,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW;IAsBzF,SAAS,CAAC,UAAU;CAIrB;AAED,qBAAa,kBAAmB,SAAQ,UAAU;IAChD,gBAAgB;IAChB,MAAM,EAAE,UAAU,EAAE,CAAC;gBAET,KAAK,EAAE,UAAU,EAAE,EAAE,WAAW,CAAC,EAAE,UAAU;IAKnD,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW;IAI5C,KAAK;CAKN;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,mBAAmB,CAAC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,qBAAa,sBAAuB,SAAQ,WAAW;IACrD,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,WAAW,CAAC,CAAwB;IAC5C,OAAO,CAAC,SAAS,CAAC,CAAa;IAE/B,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,iBAAiB,CAA8B;IACvD,OAAO,CAAC,iBAAiB,CAAkB;gBAE/B,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,kBAAkB;IAOnD,IAAI,UAAU,IAAI,OAAO,CAExB;IAEK,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;YAetC,kBAAkB;IAsChC;;OAEG;IACH,KAAK,IAAI,IAAI;IAeb,WAAW,IAAI,IAAI;YAQL,YAAY;IAkBpB,KAAK;CAIZ"}
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
} from "../../constants.js";
|
|
14
14
|
import { log } from "../../log.js";
|
|
15
15
|
import { Future, Task, shortuuid } from "../../utils.js";
|
|
16
|
-
import { AudioOutput, TextOutput } from "../io.js";
|
|
16
|
+
import { AudioOutput, TextOutput, isTimedString } from "../io.js";
|
|
17
17
|
import { findMicrophoneTrackId } from "../transcription/index.js";
|
|
18
18
|
class BaseParticipantTranscriptionOutput extends TextOutput {
|
|
19
19
|
room;
|
|
@@ -73,8 +73,9 @@ class BaseParticipantTranscriptionOutput extends TextOutput {
|
|
|
73
73
|
if (!this.participantIdentity) {
|
|
74
74
|
return;
|
|
75
75
|
}
|
|
76
|
-
|
|
77
|
-
|
|
76
|
+
const textStr = isTimedString(text) ? text.text : text;
|
|
77
|
+
this.latestText = textStr;
|
|
78
|
+
await this.handleCaptureText(textStr);
|
|
78
79
|
}
|
|
79
80
|
flush() {
|
|
80
81
|
if (!this.participantIdentity || !this.capturing) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string) {\n if (!this.participantIdentity) {\n return;\n }\n\n this.latestText = text;\n await this.handleCaptureText(text);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":"AAIA;AAAA,EAEE;AAAA,EACA;AAAA,EAKA;AAAA,EAEA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,WAAW;AACpB,SAAS,QAAQ,MAAM,iBAAiB;AACxC,SAAS,aAAa,kBAAkB;AACxC,SAAS,6BAA6B;AAEtC,MAAe,2CAA2C,WAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,SAAS,IAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,UAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,UAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,UAAU,sBAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,WAAO,UAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,IAAI;AAAA,EACnC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,6BAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,gCAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,kCAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,6BAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,gCAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAvQ7F;AAwQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,WAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAAc;AAC9B,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,YAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,OAAO;AAAA,EACzC,oBAAkC,IAAI,OAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,YAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,OAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,OAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAxalD;AAyaI,UAAM,QAAQ,gBAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oBAAoB,EAAE,QAAQ,YAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../../src/voice/room_io/_output.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { RemoteParticipant } from '@livekit/rtc-node';\nimport {\n type AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Participant,\n type RemoteTrackPublication,\n type Room,\n RoomEvent,\n type TextStreamWriter,\n TrackPublishOptions,\n TrackSource,\n} from '@livekit/rtc-node';\nimport {\n ATTRIBUTE_TRANSCRIPTION_FINAL,\n ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,\n ATTRIBUTE_TRANSCRIPTION_TRACK_ID,\n TOPIC_TRANSCRIPTION,\n} from '../../constants.js';\nimport { log } from '../../log.js';\nimport { Future, Task, shortuuid } from '../../utils.js';\nimport { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js';\nimport { findMicrophoneTrackId } from '../transcription/index.js';\n\nabstract class BaseParticipantTranscriptionOutput extends TextOutput {\n protected room: Room;\n protected isDeltaStream: boolean;\n protected participantIdentity: string | null = null;\n protected trackId?: string;\n protected capturing: boolean = false;\n protected latestText: string = '';\n protected currentId: string = this.generateCurrentId();\n protected logger = log();\n\n constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {\n super();\n this.room = room;\n this.isDeltaStream = isDeltaStream;\n\n this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);\n this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);\n\n this.setParticipant(participant);\n }\n\n setParticipant(participant: Participant | string | null) {\n if (typeof participant === 'string' || participant === null) {\n this.participantIdentity = participant;\n } else {\n this.participantIdentity = participant.identity;\n }\n\n if (!this.participantIdentity) {\n return;\n }\n\n try {\n this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);\n } catch (error) {\n // track id is optional for TextStream when audio is not published\n }\n\n this.flush();\n this.resetState();\n }\n\n protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {\n if (\n !this.participantIdentity ||\n participant.identity !== this.participantIdentity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected onLocalTrackPublished = (track: LocalTrackPublication) => {\n if (\n !this.participantIdentity ||\n this.participantIdentity !== this.room.localParticipant?.identity ||\n track.source !== TrackSource.SOURCE_MICROPHONE\n ) {\n return;\n }\n\n this.trackId = track.sid;\n };\n\n protected generateCurrentId(): string {\n return shortuuid('SG_');\n }\n\n protected resetState() {\n this.currentId = this.generateCurrentId();\n this.capturing = false;\n this.latestText = '';\n }\n\n async captureText(text: string | TimedString) {\n if (!this.participantIdentity) {\n return;\n }\n\n const textStr = isTimedString(text) ? text.text : text;\n this.latestText = textStr;\n await this.handleCaptureText(textStr);\n }\n\n flush() {\n if (!this.participantIdentity || !this.capturing) {\n return;\n }\n\n this.capturing = false;\n this.handleFlush();\n }\n\n protected abstract handleCaptureText(text: string): Promise<void>;\n protected abstract handleFlush(): void;\n}\n\nexport class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private writer: TextStreamWriter | null = null;\n private flushTask: Task<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (this.flushTask && !this.flushTask.done) {\n await this.flushTask.result;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n // reuse the existing writer\n if (this.writer === null) {\n this.writer = await this.createTextWriter();\n }\n await this.writer.write(text);\n } else {\n const tmpWriter = await this.createTextWriter();\n await tmpWriter.write(text);\n await tmpWriter.close();\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected handleFlush() {\n const currWriter = this.writer;\n this.writer = null;\n this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));\n }\n\n private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {\n if (!this.participantIdentity) {\n throw new Error('participantIdentity not found');\n }\n\n if (!this.room.localParticipant) {\n throw new Error('localParticipant not found');\n }\n\n if (!attributes) {\n attributes = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n }\n attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;\n\n return await this.room.localParticipant.streamText({\n topic: TOPIC_TRANSCRIPTION,\n senderIdentity: this.participantIdentity,\n attributes,\n });\n }\n\n private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {\n const attributes: Record<string, string> = {\n [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',\n };\n if (this.trackId) {\n attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;\n }\n\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => resolve());\n });\n\n try {\n if (this.room.isConnected) {\n if (this.isDeltaStream) {\n if (writer) {\n await Promise.race([writer.close(), abortPromise]);\n }\n } else {\n const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);\n if (signal.aborted || !tmpWriter) {\n return;\n }\n await Promise.race([tmpWriter.write(this.latestText), abortPromise]);\n if (signal.aborted) {\n return;\n }\n await Promise.race([tmpWriter.close(), abortPromise]);\n }\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n}\n\nexport class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {\n private pushedText: string = '';\n private flushTask: Promise<void> | null = null;\n\n protected async handleCaptureText(text: string): Promise<void> {\n if (!this.trackId) {\n return;\n }\n\n if (this.flushTask) {\n await this.flushTask;\n }\n\n if (!this.capturing) {\n this.resetState();\n this.capturing = true;\n }\n\n if (this.isDeltaStream) {\n this.pushedText += text;\n } else {\n this.pushedText = text;\n }\n\n await this.publishTranscription(this.currentId, this.pushedText, false);\n }\n\n protected handleFlush() {\n if (!this.trackId) {\n return;\n }\n\n this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);\n this.resetState();\n }\n\n async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {\n if (!this.participantIdentity || !this.trackId) {\n return;\n }\n\n try {\n if (this.room.isConnected) {\n if (signal?.aborted) {\n return;\n }\n\n await this.room.localParticipant?.publishTranscription({\n participantIdentity: this.participantIdentity,\n trackSid: this.trackId,\n segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],\n });\n }\n } catch (error) {\n this.logger.error(error, 'failed to publish transcription');\n }\n }\n\n protected resetState() {\n super.resetState();\n this.pushedText = '';\n }\n}\n\nexport class ParalellTextOutput extends TextOutput {\n /** @internal */\n _sinks: TextOutput[];\n\n constructor(sinks: TextOutput[], nextInChain?: TextOutput) {\n super(nextInChain);\n this._sinks = sinks;\n }\n\n async captureText(text: string | TimedString) {\n await Promise.all(this._sinks.map((sink) => sink.captureText(text)));\n }\n\n flush() {\n for (const sink of this._sinks) {\n sink.flush();\n }\n }\n}\n\nexport interface AudioOutputOptions {\n sampleRate: number;\n numChannels: number;\n trackPublishOptions: TrackPublishOptions;\n queueSizeMs?: number;\n}\nexport class ParticipantAudioOutput extends AudioOutput {\n private room: Room;\n private options: AudioOutputOptions;\n private audioSource: AudioSource;\n private publication?: LocalTrackPublication;\n private flushTask?: Task<void>;\n\n /** Duration of audio pushed to the source, in seconds */\n private pushedDuration: number = 0;\n private startedFuture: Future<void> = new Future();\n private interruptedFuture: Future<void> = new Future();\n private firstFrameEmitted: boolean = false;\n\n constructor(room: Room, options: AudioOutputOptions) {\n super(options.sampleRate, undefined, { pause: true });\n this.room = room;\n this.options = options;\n this.audioSource = new AudioSource(options.sampleRate, options.numChannels);\n }\n\n get subscribed(): boolean {\n return this.startedFuture.done;\n }\n\n async start(signal: AbortSignal): Promise<void> {\n await this.publishTrack(signal);\n }\n\n async captureFrame(frame: AudioFrame): Promise<void> {\n await this.startedFuture.await;\n\n super.captureFrame(frame);\n\n if (!this.firstFrameEmitted) {\n this.firstFrameEmitted = true;\n this.onPlaybackStarted(Date.now());\n }\n\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n await this.audioSource.captureFrame(frame);\n }\n\n private async waitForPlayoutTask(abortController: AbortController): Promise<void> {\n const abortFuture = new Future<boolean>();\n\n const resolveAbort = () => {\n if (!abortFuture.done) abortFuture.resolve(true);\n };\n\n abortController.signal.addEventListener('abort', resolveAbort);\n\n this.audioSource.waitForPlayout().finally(() => {\n abortController.signal.removeEventListener('abort', resolveAbort);\n if (!abortFuture.done) abortFuture.resolve(false);\n });\n\n const interrupted = await Promise.race([\n abortFuture.await,\n this.interruptedFuture.await.then(() => true),\n ]);\n\n let pushedDuration = this.pushedDuration;\n\n if (interrupted) {\n // Calculate actual played duration accounting for queued audio\n // Note: queuedDuration is in milliseconds, pushedDuration is in seconds\n pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);\n this.audioSource.clearQueue();\n }\n\n this.pushedDuration = 0;\n this.interruptedFuture = new Future();\n this.firstFrameEmitted = false;\n\n this.onPlaybackFinished({\n playbackPosition: pushedDuration,\n interrupted,\n });\n }\n\n /**\n * Flush any buffered audio, marking the current playback/segment as complete\n */\n flush(): void {\n super.flush();\n\n if (!this.pushedDuration) {\n return;\n }\n\n if (this.flushTask && !this.flushTask.done) {\n this.logger.error('flush called while playback is in progress');\n this.flushTask.cancel();\n }\n\n this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));\n }\n\n clearBuffer(): void {\n if (!this.pushedDuration) {\n return;\n }\n\n this.interruptedFuture.resolve();\n }\n\n private async publishTrack(signal: AbortSignal) {\n const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);\n this.publication = await this.room.localParticipant?.publishTrack(\n track,\n new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),\n );\n\n if (signal.aborted) {\n return;\n }\n\n await this.publication?.waitForSubscription();\n\n if (!this.startedFuture.done) {\n this.startedFuture.resolve();\n }\n }\n\n async close() {\n // TODO(AJS-106): add republish track\n await this.audioSource.close();\n }\n}\n"],"mappings":"AAIA;AAAA,EAEE;AAAA,EACA;AAAA,EAKA;AAAA,EAEA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,WAAW;AACpB,SAAS,QAAQ,MAAM,iBAAiB;AACxC,SAAS,aAAa,YAA8B,qBAAqB;AACzE,SAAS,6BAA6B;AAEtC,MAAe,2CAA2C,WAAW;AAAA,EACzD;AAAA,EACA;AAAA,EACA,sBAAqC;AAAA,EACrC;AAAA,EACA,YAAqB;AAAA,EACrB,aAAqB;AAAA,EACrB,YAAoB,KAAK,kBAAkB;AAAA,EAC3C,SAAS,IAAI;AAAA,EAEvB,YAAY,MAAY,eAAwB,aAA0C;AACxF,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,gBAAgB;AAErB,SAAK,KAAK,GAAG,UAAU,gBAAgB,KAAK,gBAAgB;AAC5D,SAAK,KAAK,GAAG,UAAU,qBAAqB,KAAK,qBAAqB;AAEtE,SAAK,eAAe,WAAW;AAAA,EACjC;AAAA,EAEA,eAAe,aAA0C;AACvD,QAAI,OAAO,gBAAgB,YAAY,gBAAgB,MAAM;AAC3D,WAAK,sBAAsB;AAAA,IAC7B,OAAO;AACL,WAAK,sBAAsB,YAAY;AAAA,IACzC;AAEA,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,QAAI;AACF,WAAK,UAAU,sBAAsB,KAAK,MAAM,KAAK,mBAAmB;AAAA,IAC1E,SAAS,OAAO;AAAA,IAEhB;AAEA,SAAK,MAAM;AACX,SAAK,WAAW;AAAA,EAClB;AAAA,EAEU,mBAAmB,CAAC,OAA+B,gBAAmC;AAC9F,QACE,CAAC,KAAK,uBACN,YAAY,aAAa,KAAK,uBAC9B,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,wBAAwB,CAAC,UAAiC;AAlFtE;AAmFI,QACE,CAAC,KAAK,uBACN,KAAK,0BAAwB,UAAK,KAAK,qBAAV,mBAA4B,aACzD,MAAM,WAAW,YAAY,mBAC7B;AACA;AAAA,IACF;AAEA,SAAK,UAAU,MAAM;AAAA,EACvB;AAAA,EAEU,oBAA4B;AACpC,WAAO,UAAU,KAAK;AAAA,EACxB;AAAA,EAEU,aAAa;AACrB,SAAK,YAAY,KAAK,kBAAkB;AACxC,SAAK,YAAY;AACjB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,QAAI,CAAC,KAAK,qBAAqB;AAC7B;AAAA,IACF;AAEA,UAAM,UAAU,cAAc,IAAI,IAAI,KAAK,OAAO;AAClD,SAAK,aAAa;AAClB,UAAM,KAAK,kBAAkB,OAAO;AAAA,EACtC;AAAA,EAEA,QAAQ;AACN,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,WAAW;AAChD;AAAA,IACF;AAEA,SAAK,YAAY;AACjB,SAAK,YAAY;AAAA,EACnB;AAIF;AAEO,MAAM,uCAAuC,mCAAmC;AAAA,EAC7E,SAAkC;AAAA,EAClC,YAA+B;AAAA,EAEvC,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,YAAM,KAAK,UAAU;AAAA,IACvB;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AAEtB,cAAI,KAAK,WAAW,MAAM;AACxB,iBAAK,SAAS,MAAM,KAAK,iBAAiB;AAAA,UAC5C;AACA,gBAAM,KAAK,OAAO,MAAM,IAAI;AAAA,QAC9B,OAAO;AACL,gBAAM,YAAY,MAAM,KAAK,iBAAiB;AAC9C,gBAAM,UAAU,MAAM,IAAI;AAC1B,gBAAM,UAAU,MAAM;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,cAAc;AACtB,UAAM,aAAa,KAAK;AACxB,SAAK,SAAS;AACd,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,YAAY,WAAW,MAAM,CAAC;AAAA,EAC9F;AAAA,EAEA,MAAc,iBAAiB,YAAgE;AAC7F,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AAEA,QAAI,CAAC,KAAK,KAAK,kBAAkB;AAC/B,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,QAAI,CAAC,YAAY;AACf,mBAAa;AAAA,QACX,CAAC,6BAA6B,GAAG;AAAA,MACnC;AACA,UAAI,KAAK,SAAS;AAChB,mBAAW,gCAAgC,IAAI,KAAK;AAAA,MACtD;AAAA,IACF;AACA,eAAW,kCAAkC,IAAI,KAAK;AAEtD,WAAO,MAAM,KAAK,KAAK,iBAAiB,WAAW;AAAA,MACjD,OAAO;AAAA,MACP,gBAAgB,KAAK;AAAA,MACrB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,cAAc,QAAiC,QAAoC;AAC/F,UAAM,aAAqC;AAAA,MACzC,CAAC,6BAA6B,GAAG;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,iBAAW,gCAAgC,IAAI,KAAK;AAAA,IACtD;AAEA,UAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,aAAO,iBAAiB,SAAS,MAAM,QAAQ,CAAC;AAAA,IAClD,CAAC;AAED,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,KAAK,eAAe;AACtB,cAAI,QAAQ;AACV,kBAAM,QAAQ,KAAK,CAAC,OAAO,MAAM,GAAG,YAAY,CAAC;AAAA,UACnD;AAAA,QACF,OAAO;AACL,gBAAM,YAAY,MAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,UAAU,GAAG,YAAY,CAAC;AACtF,cAAI,OAAO,WAAW,CAAC,WAAW;AAChC;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,KAAK,UAAU,GAAG,YAAY,CAAC;AACnE,cAAI,OAAO,SAAS;AAClB;AAAA,UACF;AACA,gBAAM,QAAQ,KAAK,CAAC,UAAU,MAAM,GAAG,YAAY,CAAC;AAAA,QACtD;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AACF;AAEO,MAAM,6CAA6C,mCAAmC;AAAA,EACnF,aAAqB;AAAA,EACrB,YAAkC;AAAA,EAE1C,MAAgB,kBAAkB,MAA6B;AAC7D,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,KAAK;AAAA,IACb;AAEA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,WAAW;AAChB,WAAK,YAAY;AAAA,IACnB;AAEA,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AAAA,IACrB,OAAO;AACL,WAAK,aAAa;AAAA,IACpB;AAEA,UAAM,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,KAAK;AAAA,EACxE;AAAA,EAEU,cAAc;AACtB,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,SAAK,YAAY,KAAK,qBAAqB,KAAK,WAAW,KAAK,YAAY,IAAI;AAChF,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,qBAAqB,IAAY,MAAc,OAAgB,QAAsB;AAxQ7F;AAyQI,QAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI;AACF,UAAI,KAAK,KAAK,aAAa;AACzB,YAAI,iCAAQ,SAAS;AACnB;AAAA,QACF;AAEA,gBAAM,UAAK,KAAK,qBAAV,mBAA4B,qBAAqB;AAAA,UACrD,qBAAqB,KAAK;AAAA,UAC1B,UAAU,KAAK;AAAA,UACf,UAAU,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,GAAG,UAAU,GAAG,CAAC;AAAA,QACxF;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,OAAO,iCAAiC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEU,aAAa;AACrB,UAAM,WAAW;AACjB,SAAK,aAAa;AAAA,EACpB;AACF;AAEO,MAAM,2BAA2B,WAAW;AAAA;AAAA,EAEjD;AAAA,EAEA,YAAY,OAAqB,aAA0B;AACzD,UAAM,WAAW;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,YAAY,MAA4B;AAC5C,UAAM,QAAQ,IAAI,KAAK,OAAO,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,CAAC,CAAC;AAAA,EACrE;AAAA,EAEA,QAAQ;AACN,eAAW,QAAQ,KAAK,QAAQ;AAC9B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AACF;AAQO,MAAM,+BAA+B,YAAY;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA,iBAAyB;AAAA,EACzB,gBAA8B,IAAI,OAAO;AAAA,EACzC,oBAAkC,IAAI,OAAO;AAAA,EAC7C,oBAA6B;AAAA,EAErC,YAAY,MAAY,SAA6B;AACnD,UAAM,QAAQ,YAAY,QAAW,EAAE,OAAO,KAAK,CAAC;AACpD,SAAK,OAAO;AACZ,SAAK,UAAU;AACf,SAAK,cAAc,IAAI,YAAY,QAAQ,YAAY,QAAQ,WAAW;AAAA,EAC5E;AAAA,EAEA,IAAI,aAAsB;AACxB,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAM,QAAoC;AAC9C,UAAM,KAAK,aAAa,MAAM;AAAA,EAChC;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,UAAM,KAAK,cAAc;AAEzB,UAAM,aAAa,KAAK;AAExB,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAGA,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,UAAM,KAAK,YAAY,aAAa,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAc,mBAAmB,iBAAiD;AAChF,UAAM,cAAc,IAAI,OAAgB;AAExC,UAAM,eAAe,MAAM;AACzB,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA,IACjD;AAEA,oBAAgB,OAAO,iBAAiB,SAAS,YAAY;AAE7D,SAAK,YAAY,eAAe,EAAE,QAAQ,MAAM;AAC9C,sBAAgB,OAAO,oBAAoB,SAAS,YAAY;AAChE,UAAI,CAAC,YAAY,KAAM,aAAY,QAAQ,KAAK;AAAA,IAClD,CAAC;AAED,UAAM,cAAc,MAAM,QAAQ,KAAK;AAAA,MACrC,YAAY;AAAA,MACZ,KAAK,kBAAkB,MAAM,KAAK,MAAM,IAAI;AAAA,IAC9C,CAAC;AAED,QAAI,iBAAiB,KAAK;AAE1B,QAAI,aAAa;AAGf,uBAAiB,KAAK,IAAI,KAAK,iBAAiB,KAAK,YAAY,iBAAiB,KAAM,CAAC;AACzF,WAAK,YAAY,WAAW;AAAA,IAC9B;AAEA,SAAK,iBAAiB;AACtB,SAAK,oBAAoB,IAAI,OAAO;AACpC,SAAK,oBAAoB;AAEzB,SAAK,mBAAmB;AAAA,MACtB,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,MAAM;AAC1C,WAAK,OAAO,MAAM,4CAA4C;AAC9D,WAAK,UAAU,OAAO;AAAA,IACxB;AAEA,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,mBAAmB,UAAU,CAAC;AAAA,EAChF;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,gBAAgB;AACxB;AAAA,IACF;AAEA,SAAK,kBAAkB,QAAQ;AAAA,EACjC;AAAA,EAEA,MAAc,aAAa,QAAqB;AAzalD;AA0aI,UAAM,QAAQ,gBAAgB,iBAAiB,gBAAgB,KAAK,WAAW;AAC/E,SAAK,cAAc,QAAM,UAAK,KAAK,qBAAV,mBAA4B;AAAA,MACnD;AAAA,MACA,IAAI,oBAAoB,EAAE,QAAQ,YAAY,kBAAkB,CAAC;AAAA;AAGnE,QAAI,OAAO,SAAS;AAClB;AAAA,IACF;AAEA,YAAM,UAAK,gBAAL,mBAAkB;AAExB,QAAI,CAAC,KAAK,cAAc,MAAM;AAC5B,WAAK,cAAc,QAAQ;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AAEZ,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AACF;","names":[]}
|
|
@@ -18,6 +18,7 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
18
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
19
|
var synchronizer_exports = {};
|
|
20
20
|
__export(synchronizer_exports, {
|
|
21
|
+
SpeakingRateData: () => SpeakingRateData,
|
|
21
22
|
TranscriptionSynchronizer: () => TranscriptionSynchronizer,
|
|
22
23
|
defaultTextSyncOptions: () => defaultTextSyncOptions
|
|
23
24
|
});
|
|
@@ -28,6 +29,78 @@ var import_tokenize = require("../../tokenize/index.cjs");
|
|
|
28
29
|
var import_utils = require("../../utils.cjs");
|
|
29
30
|
var import_io = require("../io.cjs");
|
|
30
31
|
const STANDARD_SPEECH_RATE = 3.83;
|
|
32
|
+
class SpeakingRateData {
|
|
33
|
+
/** Timestamps of the speaking rate. */
|
|
34
|
+
timestamps = [];
|
|
35
|
+
/** Speed at the timestamp. */
|
|
36
|
+
speakingRate = [];
|
|
37
|
+
/** Accumulated speaking units up to the timestamp. */
|
|
38
|
+
speakIntegrals = [];
|
|
39
|
+
/** Buffer for text without timing annotations yet. */
|
|
40
|
+
textBuffer = [];
|
|
41
|
+
/**
|
|
42
|
+
* Add by speaking rate estimation.
|
|
43
|
+
*/
|
|
44
|
+
addByRate(timestamp, speakingRate) {
|
|
45
|
+
const integral = this.speakIntegrals.length > 0 ? this.speakIntegrals[this.speakIntegrals.length - 1] : 0;
|
|
46
|
+
const dt = timestamp - this.pushedDuration;
|
|
47
|
+
const newIntegral = integral + speakingRate * dt;
|
|
48
|
+
this.timestamps.push(timestamp);
|
|
49
|
+
this.speakingRate.push(speakingRate);
|
|
50
|
+
this.speakIntegrals.push(newIntegral);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Add annotation from TimedString with start_time/end_time.
|
|
54
|
+
*/
|
|
55
|
+
addByAnnotation(text, startTime, endTime) {
|
|
56
|
+
if (startTime !== void 0) {
|
|
57
|
+
const integral = this.speakIntegrals.length > 0 ? this.speakIntegrals[this.speakIntegrals.length - 1] : 0;
|
|
58
|
+
const dt = startTime - this.pushedDuration;
|
|
59
|
+
const textLen = this.textBuffer.reduce((sum, t) => sum + t.length, 0);
|
|
60
|
+
const newIntegral = integral + textLen;
|
|
61
|
+
const rate = dt > 0 ? textLen / dt : 0;
|
|
62
|
+
this.timestamps.push(startTime);
|
|
63
|
+
this.speakingRate.push(rate);
|
|
64
|
+
this.speakIntegrals.push(newIntegral);
|
|
65
|
+
this.textBuffer = [];
|
|
66
|
+
}
|
|
67
|
+
this.textBuffer.push(text);
|
|
68
|
+
if (endTime !== void 0) {
|
|
69
|
+
this.addByAnnotation("", endTime, void 0);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Get accumulated speaking units up to the given timestamp.
|
|
74
|
+
*/
|
|
75
|
+
accumulateTo(timestamp) {
|
|
76
|
+
if (this.timestamps.length === 0) {
|
|
77
|
+
return 0;
|
|
78
|
+
}
|
|
79
|
+
let idx = 0;
|
|
80
|
+
for (let i = 0; i < this.timestamps.length; i++) {
|
|
81
|
+
if (this.timestamps[i] <= timestamp) {
|
|
82
|
+
idx = i + 1;
|
|
83
|
+
} else {
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (idx === 0) {
|
|
88
|
+
return 0;
|
|
89
|
+
}
|
|
90
|
+
let integralT = this.speakIntegrals[idx - 1];
|
|
91
|
+
const dt = timestamp - this.timestamps[idx - 1];
|
|
92
|
+
const rate = idx < this.speakingRate.length ? this.speakingRate[idx] : this.speakingRate[idx - 1];
|
|
93
|
+
integralT += rate * dt;
|
|
94
|
+
if (idx < this.timestamps.length) {
|
|
95
|
+
integralT = Math.min(integralT, this.speakIntegrals[idx]);
|
|
96
|
+
}
|
|
97
|
+
return integralT;
|
|
98
|
+
}
|
|
99
|
+
/** Get the last pushed timestamp. */
|
|
100
|
+
get pushedDuration() {
|
|
101
|
+
return this.timestamps.length > 0 ? this.timestamps[this.timestamps.length - 1] : 0;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
31
104
|
class SegmentSynchronizerImpl {
|
|
32
105
|
constructor(options, nextInChain) {
|
|
33
106
|
this.options = options;
|
|
@@ -42,7 +115,8 @@ class SegmentSynchronizerImpl {
|
|
|
42
115
|
};
|
|
43
116
|
this.audioData = {
|
|
44
117
|
pushedDuration: 0,
|
|
45
|
-
done: false
|
|
118
|
+
done: false,
|
|
119
|
+
annotatedRate: null
|
|
46
120
|
};
|
|
47
121
|
this.outputStream = new import_identity_transform.IdentityTransform();
|
|
48
122
|
this.outputStreamWriter = this.outputStream.writable.getWriter();
|
|
@@ -73,6 +147,9 @@ class SegmentSynchronizerImpl {
|
|
|
73
147
|
get textInputEnded() {
|
|
74
148
|
return this.textData.done;
|
|
75
149
|
}
|
|
150
|
+
get hasPendingText() {
|
|
151
|
+
return this.textData.pushedText.length > this.textData.forwardedText.length;
|
|
152
|
+
}
|
|
76
153
|
get readable() {
|
|
77
154
|
return this.outputStream.readable;
|
|
78
155
|
}
|
|
@@ -100,8 +177,22 @@ class SegmentSynchronizerImpl {
|
|
|
100
177
|
this.logger.warn("SegmentSynchronizerImpl.pushText called after close");
|
|
101
178
|
return;
|
|
102
179
|
}
|
|
103
|
-
|
|
104
|
-
|
|
180
|
+
let textStr;
|
|
181
|
+
let startTime;
|
|
182
|
+
let endTime;
|
|
183
|
+
if ((0, import_io.isTimedString)(text)) {
|
|
184
|
+
textStr = text.text;
|
|
185
|
+
startTime = text.startTime;
|
|
186
|
+
endTime = text.endTime;
|
|
187
|
+
if (!this.audioData.annotatedRate) {
|
|
188
|
+
this.audioData.annotatedRate = new SpeakingRateData();
|
|
189
|
+
}
|
|
190
|
+
this.audioData.annotatedRate.addByAnnotation(textStr, startTime, endTime);
|
|
191
|
+
} else {
|
|
192
|
+
textStr = text;
|
|
193
|
+
}
|
|
194
|
+
this.textData.sentenceStream.pushText(textStr);
|
|
195
|
+
this.textData.pushedText += textStr;
|
|
105
196
|
}
|
|
106
197
|
endTextInput() {
|
|
107
198
|
if (this.closed) {
|
|
@@ -121,6 +212,9 @@ class SegmentSynchronizerImpl {
|
|
|
121
212
|
{ textDone: this.textData.done, audioDone: this.audioData.done },
|
|
122
213
|
"SegmentSynchronizerImpl.markPlaybackFinished called before text/audio input is done"
|
|
123
214
|
);
|
|
215
|
+
if (!interrupted) {
|
|
216
|
+
this.playbackCompleted = true;
|
|
217
|
+
}
|
|
124
218
|
return;
|
|
125
219
|
}
|
|
126
220
|
if (!interrupted) {
|
|
@@ -140,7 +234,6 @@ class SegmentSynchronizerImpl {
|
|
|
140
234
|
if (done) {
|
|
141
235
|
break;
|
|
142
236
|
}
|
|
143
|
-
this.textData.forwardedText += text;
|
|
144
237
|
await this.nextInChain.captureText(text);
|
|
145
238
|
}
|
|
146
239
|
reader.releaseLock();
|
|
@@ -171,16 +264,32 @@ class SegmentSynchronizerImpl {
|
|
|
171
264
|
}
|
|
172
265
|
const wordHphens = this.options.hyphenateWord(word).length;
|
|
173
266
|
const elapsedSeconds = (Date.now() - this.startWallTime) / 1e3;
|
|
174
|
-
|
|
175
|
-
const
|
|
176
|
-
|
|
267
|
+
let dHyphens = 0;
|
|
268
|
+
const annotated = this.audioData.annotatedRate;
|
|
269
|
+
if (annotated && annotated.pushedDuration >= elapsedSeconds) {
|
|
270
|
+
const targetLen = Math.floor(annotated.accumulateTo(elapsedSeconds));
|
|
271
|
+
const forwardedLen = this.textData.forwardedText.length;
|
|
272
|
+
if (targetLen >= forwardedLen) {
|
|
273
|
+
const dText = this.textData.pushedText.slice(forwardedLen, targetLen);
|
|
274
|
+
dHyphens = this.calcHyphens(dText).length;
|
|
275
|
+
} else {
|
|
276
|
+
const dText = this.textData.pushedText.slice(targetLen, forwardedLen);
|
|
277
|
+
dHyphens = -this.calcHyphens(dText).length;
|
|
278
|
+
}
|
|
279
|
+
} else {
|
|
280
|
+
const targetHyphens = elapsedSeconds * this.options.speed;
|
|
281
|
+
dHyphens = Math.max(0, targetHyphens - this.textData.forwardedHyphens);
|
|
282
|
+
}
|
|
283
|
+
let delayTime = Math.max(0, wordHphens - dHyphens) / this.speed;
|
|
177
284
|
if (this.playbackCompleted) {
|
|
178
|
-
|
|
285
|
+
delayTime = 0;
|
|
179
286
|
}
|
|
180
|
-
await this.sleepIfNotClosed(
|
|
181
|
-
|
|
182
|
-
|
|
287
|
+
await this.sleepIfNotClosed(delayTime / 2);
|
|
288
|
+
const forwardedWord = sentence.slice(textCursor, endPos);
|
|
289
|
+
this.outputStreamWriter.write(forwardedWord);
|
|
290
|
+
await this.sleepIfNotClosed(delayTime / 2);
|
|
183
291
|
this.textData.forwardedHyphens += wordHphens;
|
|
292
|
+
this.textData.forwardedText += forwardedWord;
|
|
184
293
|
textCursor = endPos;
|
|
185
294
|
}
|
|
186
295
|
if (textCursor < sentence.length) {
|
|
@@ -189,6 +298,14 @@ class SegmentSynchronizerImpl {
|
|
|
189
298
|
}
|
|
190
299
|
}
|
|
191
300
|
}
|
|
301
|
+
calcHyphens(text) {
|
|
302
|
+
const words = this.options.splitWords(text);
|
|
303
|
+
const hyphens = [];
|
|
304
|
+
for (const [word] of words) {
|
|
305
|
+
hyphens.push(...this.options.hyphenateWord(word));
|
|
306
|
+
}
|
|
307
|
+
return hyphens;
|
|
308
|
+
}
|
|
192
309
|
async sleepIfNotClosed(sleepTimeSeconds) {
|
|
193
310
|
if (this.closed) {
|
|
194
311
|
return;
|
|
@@ -311,6 +428,10 @@ class SyncedAudioOutput extends import_io.AudioOutput {
|
|
|
311
428
|
return;
|
|
312
429
|
}
|
|
313
430
|
if (!this.pushedDuration) {
|
|
431
|
+
if (this.synchronizer._impl.hasPendingText) {
|
|
432
|
+
this.synchronizer._impl.endAudioInput();
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
314
435
|
this.synchronizer.rotateSegment();
|
|
315
436
|
return;
|
|
316
437
|
}
|
|
@@ -345,8 +466,9 @@ class SyncedTextOutput extends import_io.TextOutput {
|
|
|
345
466
|
logger = (0, import_log.log)();
|
|
346
467
|
async captureText(text) {
|
|
347
468
|
await this.synchronizer.barrier();
|
|
469
|
+
const textStr = (0, import_io.isTimedString)(text) ? text.text : text;
|
|
348
470
|
if (!this.synchronizer.enabled) {
|
|
349
|
-
await this.nextInChain.captureText(
|
|
471
|
+
await this.nextInChain.captureText(textStr);
|
|
350
472
|
return;
|
|
351
473
|
}
|
|
352
474
|
this.capturing = true;
|
|
@@ -359,7 +481,8 @@ class SyncedTextOutput extends import_io.TextOutput {
|
|
|
359
481
|
}
|
|
360
482
|
this.synchronizer._impl.pushText(text);
|
|
361
483
|
}
|
|
362
|
-
flush() {
|
|
484
|
+
async flush() {
|
|
485
|
+
await this.synchronizer.barrier();
|
|
363
486
|
if (!this.synchronizer.enabled) {
|
|
364
487
|
this.nextInChain.flush();
|
|
365
488
|
return;
|
|
@@ -373,6 +496,7 @@ class SyncedTextOutput extends import_io.TextOutput {
|
|
|
373
496
|
}
|
|
374
497
|
// Annotate the CommonJS export names for ESM import in node:
|
|
375
498
|
0 && (module.exports = {
|
|
499
|
+
SpeakingRateData,
|
|
376
500
|
TranscriptionSynchronizer,
|
|
377
501
|
defaultTextSyncOptions
|
|
378
502
|
});
|