@whereby.com/assistant-sdk 0.0.0-canary-20250916072551 → 0.0.0-canary-20250917154617

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/tools.cjs CHANGED
@@ -28,301 +28,362 @@ const STREAM_INPUT_SAMPLE_RATE_IN_HZ = 48000;
28
28
  const BYTES_PER_SAMPLE = 2;
29
29
  // 480 samples per 10ms frame at 48kHz
30
30
  const FRAME_10MS_SAMPLES = 480;
31
- const slotBuffers = new Map();
32
- function appendAndDrainTo480(slot, newSamples) {
33
- var _a;
34
- const prev = (_a = slotBuffers.get(slot)) !== null && _a !== void 0 ? _a : new Int16Array(0);
35
- const merged = new Int16Array(prev.length + newSamples.length);
36
- merged.set(prev, 0);
37
- merged.set(newSamples, prev.length);
38
- let offset = 0;
39
- while (merged.length - offset >= FRAME_10MS_SAMPLES) {
40
- const chunk = merged.subarray(offset, offset + FRAME_10MS_SAMPLES);
41
- enqueueFrame(slot, chunk); // always 480
42
- offset += FRAME_10MS_SAMPLES;
43
- }
44
- slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
45
- }
46
- ({
47
- enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
48
- enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
49
- wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
50
- wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
51
- lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
52
- });
53
- let slots = [];
54
- let stopPacerFn = null;
55
- let outputPacerState = null;
56
- /**
57
- * Simple linear interpolation resampler to convert audio to 48kHz.
58
- * This handles the common case of 16kHz -> 48kHz (3x upsampling).
59
- */
60
- function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
61
- const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
62
- const outputLength = Math.floor(inputFrames * ratio);
63
- const output = new Int16Array(outputLength);
64
- for (let i = 0; i < outputLength; i++) {
65
- const inputIndex = i / ratio;
66
- const index = Math.floor(inputIndex);
67
- const fraction = inputIndex - index;
68
- if (index + 1 < inputSamples.length) {
69
- const sample1 = inputSamples[index];
70
- const sample2 = inputSamples[index + 1];
71
- output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
72
- }
73
- else {
74
- output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
31
+ function createFfmpegMixer() {
32
+ const slotBuffers = new Map();
33
+ function appendAndDrainTo480(slot, newSamples) {
34
+ var _a;
35
+ const prev = (_a = slotBuffers.get(slot)) !== null && _a !== void 0 ? _a : new Int16Array(0);
36
+ const merged = new Int16Array(prev.length + newSamples.length);
37
+ merged.set(prev, 0);
38
+ merged.set(newSamples, prev.length);
39
+ let offset = 0;
40
+ while (merged.length - offset >= FRAME_10MS_SAMPLES) {
41
+ const chunk = merged.subarray(offset, offset + FRAME_10MS_SAMPLES);
42
+ enqueueFrame(slot, chunk); // always 480
43
+ offset += FRAME_10MS_SAMPLES;
75
44
  }
45
+ slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
76
46
  }
77
- return output;
78
- }
79
- /**
80
- * Enqueue an audio frame for paced delivery to the RTCAudioSource.
81
- */
82
- function enqueueOutputFrame(samples) {
83
- if (outputPacerState) {
84
- outputPacerState.frameQueue.push(samples);
47
+ ({
48
+ enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
49
+ enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
50
+ wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
51
+ wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
52
+ lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
53
+ });
54
+ let slots = [];
55
+ let stopPacerFn = null;
56
+ let outputPacerState = null;
57
+ /**
58
+ * Simple linear interpolation resampler to convert audio to 48kHz.
59
+ * This handles the common case of 16kHz -> 48kHz (3x upsampling).
60
+ */
61
+ function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
62
+ const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
63
+ const outputLength = Math.floor(inputFrames * ratio);
64
+ const output = new Int16Array(outputLength);
65
+ for (let i = 0; i < outputLength; i++) {
66
+ const inputIndex = i / ratio;
67
+ const index = Math.floor(inputIndex);
68
+ const fraction = inputIndex - index;
69
+ if (index + 1 < inputSamples.length) {
70
+ const sample1 = inputSamples[index];
71
+ const sample2 = inputSamples[index + 1];
72
+ output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
73
+ }
74
+ else {
75
+ output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
76
+ }
77
+ }
78
+ return output;
85
79
  }
86
- }
87
- /**
88
- * Start the audio pacer loop for all input slots in an FFmpeg process.
89
- *
90
- * The pacer ensures each slot (pipe:3..3+N-1) is written to at a steady
91
- * real-time rate (e.g. 10 ms = 480 samples @ 48kHz), even if WebRTC frames
92
- * arrive jittery, bursty, or with slightly different clocks.
93
- *
94
- * Key behavior:
95
- * - Writes exactly one frame per period, on a shared wall-clock grid.
96
- * - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
97
- * never stalls.
98
- * - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
99
- * - Honors Node stream backpressure (`write()` return false) without breaking
100
- * the timing grid.
101
- *
102
- * This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
103
- * can mix them without slow-downs or drift.
104
- *
105
- * Call this once right after spawning FFmpeg:
106
- * ```ts
107
- * const ff = spawnFFmpegProcess();
108
- * startPacer(ff, PARTICIPANT_SLOTS);
109
- * ```
110
- *
111
- * When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
112
- *
113
- * @param ff Child process handle from spawn("ffmpeg", ...)
114
- * @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
115
- */
116
- function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
117
- if (stopPacerFn) {
118
- stopPacerFn();
119
- stopPacerFn = null;
80
+ /**
81
+ * Enqueue an audio frame for paced delivery to the RTCAudioSource.
82
+ */
83
+ function enqueueOutputFrame(samples) {
84
+ if (outputPacerState) {
85
+ outputPacerState.frameQueue.push(samples);
86
+ }
120
87
  }
121
- const writers = Array.from({ length: slotCount }, (_, i) => ff.stdio[3 + i]);
122
- const nowMs = () => Number(process.hrtime.bigint()) / 1e6;
123
- const outputFrameMs = (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms
124
- const t0 = nowMs();
125
- slots = Array.from({ length: slotCount }, () => ({
126
- q: [],
127
- lastFrames: FRAME_10MS_SAMPLES, // keep constant
128
- nextDueMs: t0 + (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000,
129
- }));
130
- outputPacerState = {
131
- frameQueue: [],
132
- nextDueMs: t0 + outputFrameMs,
133
- rtcAudioSource,
134
- onAudioStreamReady,
135
- didEmitReadyEvent: false,
136
- };
137
- const iv = setInterval(() => {
138
- const t = nowMs();
139
- for (let s = 0; s < slotCount; s++) {
140
- const st = slots[s];
141
- const w = writers[s];
142
- const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms if 480, 20ms if 960
143
- if (t >= st.nextDueMs) {
144
- const buf = st.q.length ? st.q.shift() : Buffer.alloc(st.lastFrames * BYTES_PER_SAMPLE);
145
- if (!w.write(buf)) {
146
- // Just continue without adding drain listener - backpressure will naturally resolve
88
+ /**
89
+ * Start the audio pacer loop for all input slots in an FFmpeg process.
90
+ *
91
+ * The pacer ensures each slot (pipe:3..3+N-1) is written to at a steady
92
+ * real-time rate (e.g. 10 ms = 480 samples @ 48kHz), even if WebRTC frames
93
+ * arrive jittery, bursty, or with slightly different clocks.
94
+ *
95
+ * Key behavior:
96
+ * - Writes exactly one frame per period, on a shared wall-clock grid.
97
+ * - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
98
+ * never stalls.
99
+ * - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
100
+ * - Honors Node stream backpressure (`write()` return false) without breaking
101
+ * the timing grid.
102
+ *
103
+ * This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
104
+ * can mix them without slow-downs or drift.
105
+ *
106
+ * Call this once right after spawning FFmpeg:
107
+ * ```ts
108
+ * const ff = spawnFFmpegProcess();
109
+ * startPacer(ff, PARTICIPANT_SLOTS);
110
+ * ```
111
+ *
112
+ * When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
113
+ *
114
+ * @param ff Child process handle from spawn("ffmpeg", ...)
115
+ * @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
116
+ */
117
+ function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
118
+ if (stopPacerFn) {
119
+ stopPacerFn();
120
+ stopPacerFn = null;
121
+ }
122
+ const writers = Array.from({ length: slotCount }, (_, i) => ff.stdio[3 + i]);
123
+ const nowMs = () => Number(process.hrtime.bigint()) / 1e6;
124
+ const outputFrameMs = (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms
125
+ const t0 = nowMs();
126
+ slots = Array.from({ length: slotCount }, () => ({
127
+ q: [],
128
+ lastFrames: FRAME_10MS_SAMPLES, // keep constant
129
+ nextDueMs: t0 + (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000,
130
+ }));
131
+ outputPacerState = {
132
+ frameQueue: [],
133
+ nextDueMs: t0 + outputFrameMs,
134
+ rtcAudioSource,
135
+ onAudioStreamReady,
136
+ didEmitReadyEvent: false,
137
+ };
138
+ const iv = setInterval(() => {
139
+ const t = nowMs();
140
+ for (let s = 0; s < slotCount; s++) {
141
+ const st = slots[s];
142
+ const w = writers[s];
143
+ const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms if 480, 20ms if 960
144
+ if (t >= st.nextDueMs) {
145
+ const buf = st.q.length ? st.q.shift() : Buffer.alloc(st.lastFrames * BYTES_PER_SAMPLE);
146
+ if (!w.write(buf)) {
147
+ // Just continue without adding drain listener - backpressure will naturally resolve
148
+ const late = t - st.nextDueMs;
149
+ const steps = Math.max(1, Math.ceil(late / frameMs));
150
+ st.nextDueMs += steps * frameMs;
151
+ continue;
152
+ }
147
153
  const late = t - st.nextDueMs;
148
154
  const steps = Math.max(1, Math.ceil(late / frameMs));
149
155
  st.nextDueMs += steps * frameMs;
150
- continue;
151
156
  }
152
- const late = t - st.nextDueMs;
153
- const steps = Math.max(1, Math.ceil(late / frameMs));
154
- st.nextDueMs += steps * frameMs;
155
157
  }
156
- }
157
- if (!outputPacerState)
158
- return;
159
- // Handle output pacer for RTCAudioSource
160
- const state = outputPacerState;
161
- if (t >= state.nextDueMs) {
162
- const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
163
- if (!state.didEmitReadyEvent) {
164
- state.onAudioStreamReady();
165
- state.didEmitReadyEvent = true;
158
+ if (!outputPacerState)
159
+ return;
160
+ // Handle output pacer for RTCAudioSource
161
+ const state = outputPacerState;
162
+ if (t >= state.nextDueMs) {
163
+ const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
164
+ if (!state.didEmitReadyEvent) {
165
+ state.onAudioStreamReady();
166
+ state.didEmitReadyEvent = true;
167
+ }
168
+ state.rtcAudioSource.onData({
169
+ samples: samples,
170
+ sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
171
+ });
172
+ const late = t - state.nextDueMs;
173
+ const steps = Math.max(1, Math.ceil(late / outputFrameMs));
174
+ state.nextDueMs += steps * outputFrameMs;
166
175
  }
167
- state.rtcAudioSource.onData({
168
- samples: samples,
169
- sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
170
- });
171
- const late = t - state.nextDueMs;
172
- const steps = Math.max(1, Math.ceil(late / outputFrameMs));
173
- state.nextDueMs += steps * outputFrameMs;
174
- }
175
- }, 5);
176
- stopPacerFn = () => clearInterval(iv);
177
- }
178
- /**
179
- * Stop the audio pacer loop and clear all input slots.
180
- * Call this before killing the FFmpeg process to ensure clean shutdown.
181
- */
182
- function stopPacer() {
183
- if (stopPacerFn)
184
- stopPacerFn();
185
- stopPacerFn = null;
186
- slots = [];
187
- }
188
- /**
189
- * Queue a live frame for a given slot (0..N-1).
190
- * Auto-resnaps the slot's schedule if the frame size (480/960) changes.
191
- */
192
- function enqueueFrame(slot, samples, numberOfFrames) {
193
- const st = slots[slot];
194
- if (!st)
195
- return;
196
- const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
197
- st.q.push(buf);
198
- }
199
- /**
200
- * Clear the audio queue for a specific slot when a participant leaves.
201
- * This prevents stale audio data from continuing to play after disconnect.
202
- */
203
- function clearSlotQueue(slot) {
204
- const st = slots[slot];
205
- if (st) {
206
- st.q = [];
207
- const now = Number(process.hrtime.bigint()) / 1e6;
208
- const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
209
- st.nextDueMs = now + frameMs;
176
+ }, 5);
177
+ stopPacerFn = () => clearInterval(iv);
210
178
  }
211
- }
212
- /**
213
- * Get the FFmpeg arguments for mixing audio from multiple participants.
214
- * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
215
- * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
216
- */
217
- function getFFmpegArguments() {
218
- const N = PARTICIPANT_SLOTS;
219
- const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
220
- const ffArgs = [];
221
- for (let i = 0; i < N; i++) {
222
- ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
179
+ /**
180
+ * Stop the audio pacer loop and clear all input slots.
181
+ * Call this before killing the FFmpeg process to ensure clean shutdown.
182
+ */
183
+ function stopPacer() {
184
+ if (stopPacerFn)
185
+ stopPacerFn();
186
+ stopPacerFn = null;
187
+ slots = [];
188
+ slotBuffers.clear();
189
+ outputPacerState = null;
223
190
  }
224
- const pre = [];
225
- for (let i = 0; i < N; i++) {
226
- pre.push(`[${i}:a]aresample=async=1:first_pts=0,asetpts=N/SR/TB[a${i}]`);
191
+ /**
192
+ * Queue a live frame for a given slot (0..N-1).
193
+ * Auto-resnaps the slot's schedule if the frame size (480/960) changes.
194
+ */
195
+ function enqueueFrame(slot, samples, numberOfFrames) {
196
+ const st = slots[slot];
197
+ if (!st)
198
+ return;
199
+ const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
200
+ st.q.push(buf);
227
201
  }
228
- const labels = Array.from({ length: N }, (_, i) => `[a${i}]`).join("");
229
- const amix = `${labels}amix=inputs=${N}:duration=longest:dropout_transition=250:normalize=0[mix]`;
230
- const filter = `${pre.join(";")};${amix}`;
231
- ffArgs.push("-hide_banner", "-nostats", "-loglevel", "error", "-filter_complex", filter, "-map", "[mix]", "-f", "s16le", "-ar", String(SR), "-ac", "1", "-c:a", "pcm_s16le", "pipe:1");
232
- return ffArgs;
233
- }
234
- /**
235
- * Spawn a new FFmpeg process for mixing audio from multiple participants.
236
- * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
237
- * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
238
- * The process will log its output to stderr.
239
- * @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
240
- * @return The spawned FFmpeg process.
241
- */
242
- function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
243
- const stdio = ["ignore", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
244
- const args = getFFmpegArguments();
245
- const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
246
- startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
247
- ffmpegProcess.stderr.setEncoding("utf8");
248
- ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
249
- ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
250
- let audioBuffer = Buffer.alloc(0);
251
- const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
252
- ffmpegProcess.stdout.on("data", (chunk) => {
253
- audioBuffer = Buffer.concat([audioBuffer, chunk]);
254
- while (audioBuffer.length >= FRAME_SIZE_BYTES) {
255
- const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
256
- const samples = new Int16Array(FRAME_10MS_SAMPLES);
257
- for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
258
- samples[i] = frameData.readInt16LE(i * 2);
259
- }
260
- enqueueOutputFrame(samples);
261
- audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
202
+ /**
203
+ * Clear the audio queue for a specific slot when a participant leaves.
204
+ * This prevents stale audio data from continuing to play after disconnect.
205
+ */
206
+ function clearSlotQueue(slot) {
207
+ const st = slots[slot];
208
+ if (st) {
209
+ st.q = [];
210
+ slotBuffers.delete(slot);
211
+ const now = Number(process.hrtime.bigint()) / 1e6;
212
+ const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
213
+ st.nextDueMs = now + frameMs;
262
214
  }
263
- });
264
- return ffmpegProcess;
265
- }
266
- /**
267
- * Write audio data from a MediaStreamTrack to the FFmpeg process.
268
- * This function creates an AudioSink for the track and sets up a data handler
269
- * that enqueues audio frames into the pacer.
270
- *
271
- * @param ffmpegProcess The FFmpeg process to which audio data will be written.
272
- * @param slot The participant slot number (0..N-1) to which this track belongs.
273
- * @param audioTrack The MediaStreamTrack containing the audio data.
274
- * @return An object containing the AudioSink, the writable stream, and a stop function.
275
- */
276
- function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
277
- const writer = ffmpegProcess.stdio[3 + slot];
278
- const sink = new AudioSink(audioTrack);
279
- const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
280
- if (ch !== 1 || bitsPerSample !== 16)
281
- return;
282
- let out = samples;
283
- if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
284
- const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
285
- out = resampled;
215
+ }
216
+ /**
217
+ * Get the FFmpeg arguments for debugging, which writes each participant's audio to a separate WAV file
218
+ * and also mixes them into a single WAV file.
219
+ * This is useful for inspecting the audio quality and timing of each participant.
220
+ */
221
+ function getFFmpegArgumentsDebug() {
222
+ const N = PARTICIPANT_SLOTS;
223
+ const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
224
+ const ffArgs = [];
225
+ for (let i = 0; i < N; i++) {
226
+ ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
286
227
  }
287
- appendAndDrainTo480(slot, out);
288
- });
289
- const stop = () => {
290
- try {
291
- unsubscribe();
292
- sink.stop();
228
+ const pre = [];
229
+ for (let i = 0; i < N; i++) {
230
+ pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS,asplit=2[a${i}tap][a${i}mix]`);
293
231
  }
294
- catch (_a) {
295
- console.error("Failed to stop AudioSink");
232
+ const mixInputs = Array.from({ length: N }, (_, i) => `[a${i}mix]`).join("");
233
+ const filter = `${pre.join(";")};${mixInputs}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
234
+ ffArgs.push("-hide_banner", "-nostats", "-loglevel", "info", "-y", "-filter_complex", filter);
235
+ for (let i = 0; i < N; i++) {
236
+ ffArgs.push("-map", `[a${i}tap]`, "-f", "wav", "-c:a", "pcm_s16le", `pre${i}.wav`);
296
237
  }
297
- };
298
- return { sink, writer, stop };
299
- }
300
- /**
301
- * Stop the FFmpeg process and clean up all resources.
302
- * This function will unpipe the stdout, end all writable streams for each participant slot,
303
- * and kill the FFmpeg process.
304
- * @param ffmpegProcess The FFmpeg process to stop.
305
- */
306
- function stopFFmpegProcess(ffmpegProcess) {
307
- stopPacer();
308
- if (ffmpegProcess && !ffmpegProcess.killed) {
309
- try {
310
- ffmpegProcess.stdout.unpipe();
238
+ ffArgs.push("-map", "[mix]", "-f", "wav", "-c:a", "pcm_s16le", "mixed.wav");
239
+ return ffArgs;
240
+ }
241
+ /**
242
+ * Get the FFmpeg arguments for mixing audio from multiple participants.
243
+ * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
244
+ * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
245
+ */
246
+ function getFFmpegArguments() {
247
+ const N = PARTICIPANT_SLOTS;
248
+ const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
249
+ const ffArgs = [];
250
+ for (let i = 0; i < N; i++) {
251
+ ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
311
252
  }
312
- catch (_a) {
313
- console.error("Failed to unpipe ffmpeg stdout");
253
+ const pre = [];
254
+ for (let i = 0; i < N; i++) {
255
+ pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS[a${i}]`);
314
256
  }
315
- for (let i = 0; i < PARTICIPANT_SLOTS; i++) {
316
- const w = ffmpegProcess.stdio[3 + i];
257
+ const labels = Array.from({ length: N }, (_, i) => `[a${i}]`).join("");
258
+ const amix = `${labels}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
259
+ const filter = `${pre.join(";")};${amix}`;
260
+ ffArgs.push("-hide_banner", "-nostats", "-loglevel", "error", "-filter_complex", filter, "-map", "[mix]", "-f", "s16le", "-ar", String(SR), "-ac", "1", "-c:a", "pcm_s16le", "pipe:1");
261
+ return ffArgs;
262
+ }
263
+ /*
264
+ * Spawn a new FFmpeg process for debugging purposes.
265
+ * This will write each participant's audio to a separate WAV file and also mix them into a single WAV file.
266
+ * The output files will be named pre0.wav, pre1.wav, ..., and mixed.wav.
267
+ * The process will log its output to stderr.
268
+ * @return The spawned FFmpeg process.
269
+ */
270
+ function spawnFFmpegProcessDebug(rtcAudioSource, onAudioStreamReady) {
271
+ const stdio = ["ignore", "ignore", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
272
+ const args = getFFmpegArgumentsDebug();
273
+ const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
274
+ startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
275
+ ffmpegProcess.stderr.setEncoding("utf8");
276
+ ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
277
+ ffmpegProcess.on("error", () => console.error("FFmpeg process error (debug): is ffmpeg installed?"));
278
+ return ffmpegProcess;
279
+ }
280
+ /**
281
+ * Spawn a new FFmpeg process for mixing audio from multiple participants.
282
+ * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
283
+ * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
284
+ * The process will log its output to stderr.
285
+ * @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
286
+ * @return The spawned FFmpeg process.
287
+ */
288
+ function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
289
+ const stdio = ["pipe", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
290
+ const args = getFFmpegArguments();
291
+ const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
292
+ startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
293
+ ffmpegProcess.stderr.setEncoding("utf8");
294
+ ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
295
+ ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
296
+ let audioBuffer = Buffer.alloc(0);
297
+ const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
298
+ ffmpegProcess.stdout.on("data", (chunk) => {
299
+ audioBuffer = Buffer.concat([audioBuffer, chunk]);
300
+ while (audioBuffer.length >= FRAME_SIZE_BYTES) {
301
+ const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
302
+ const samples = new Int16Array(FRAME_10MS_SAMPLES);
303
+ for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
304
+ samples[i] = frameData.readInt16LE(i * 2);
305
+ }
306
+ enqueueOutputFrame(samples);
307
+ audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
308
+ }
309
+ });
310
+ return ffmpegProcess;
311
+ }
312
+ /**
313
+ * Write audio data from a MediaStreamTrack to the FFmpeg process.
314
+ * This function creates an AudioSink for the track and sets up a data handler
315
+ * that enqueues audio frames into the pacer.
316
+ *
317
+ * @param ffmpegProcess The FFmpeg process to which audio data will be written.
318
+ * @param slot The participant slot number (0..N-1) to which this track belongs.
319
+ * @param audioTrack The MediaStreamTrack containing the audio data.
320
+ * @return An object containing the AudioSink, the writable stream, and a stop function.
321
+ */
322
+ function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
323
+ const writer = ffmpegProcess.stdio[3 + slot];
324
+ const sink = new AudioSink(audioTrack);
325
+ const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
326
+ if (ch !== 1 || bitsPerSample !== 16)
327
+ return;
328
+ let out = samples;
329
+ if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
330
+ const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
331
+ out = resampled;
332
+ }
333
+ appendAndDrainTo480(slot, out);
334
+ });
335
+ const stop = () => {
317
336
  try {
318
- w.end();
337
+ unsubscribe();
338
+ sink.stop();
319
339
  }
320
- catch (_b) {
321
- console.error("Failed to end ffmpeg writable stream");
340
+ catch (_a) {
341
+ console.error("Failed to stop AudioSink");
342
+ }
343
+ };
344
+ return { sink, writer, stop };
345
+ }
346
+ /**
347
+ * Stop the FFmpeg process and clean up all resources.
348
+ * This function will unpipe the stdout, end all writable streams for each participant slot,
349
+ * and kill the FFmpeg process.
350
+ * @param ffmpegProcess The FFmpeg process to stop.
351
+ */
352
+ function stopFFmpegProcess(ffmpegProcess) {
353
+ var _a, _b;
354
+ stopPacer();
355
+ if (ffmpegProcess && !ffmpegProcess.killed) {
356
+ try {
357
+ ffmpegProcess.stdout.unpipe();
358
+ }
359
+ catch (_c) {
360
+ console.error("Failed to unpipe ffmpeg stdout");
361
+ }
362
+ for (let i = 0; i < PARTICIPANT_SLOTS; i++) {
363
+ const w = ffmpegProcess.stdio[3 + i];
364
+ try {
365
+ w.end();
366
+ }
367
+ catch (_d) {
368
+ console.error("Failed to end ffmpeg writable stream");
369
+ }
370
+ }
371
+ try {
372
+ (_a = ffmpegProcess.stdin) === null || _a === void 0 ? void 0 : _a.write("q\n");
373
+ (_b = ffmpegProcess.stdin) === null || _b === void 0 ? void 0 : _b.end();
374
+ }
375
+ catch (_e) {
376
+ console.error("Failed to end ffmpeg stdin");
322
377
  }
323
378
  }
324
- ffmpegProcess.kill("SIGTERM");
325
379
  }
380
+ return {
381
+ spawnFFmpegProcess,
382
+ spawnFFmpegProcessDebug,
383
+ writeAudioDataToFFmpeg,
384
+ stopFFmpegProcess,
385
+ clearSlotQueue,
386
+ };
326
387
  }
327
388
 
328
389
  class AudioMixer extends events.EventEmitter {
@@ -333,6 +394,7 @@ class AudioMixer extends events.EventEmitter {
333
394
  this.rtcAudioSource = null;
334
395
  this.participantSlots = new Map();
335
396
  this.activeSlots = {};
397
+ this.mixer = createFfmpegMixer();
336
398
  this.setupMediaStream();
337
399
  this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
338
400
  this.onStreamReady = onStreamReady;
@@ -351,7 +413,7 @@ class AudioMixer extends events.EventEmitter {
351
413
  return;
352
414
  }
353
415
  if (!this.ffmpegProcess && this.rtcAudioSource) {
354
- this.ffmpegProcess = spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
416
+ this.ffmpegProcess = this.mixer.spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
355
417
  }
356
418
  for (const p of participants)
357
419
  this.attachParticipantIfNeeded(p);
@@ -364,7 +426,7 @@ class AudioMixer extends events.EventEmitter {
364
426
  }
365
427
  stopAudioMixer() {
366
428
  if (this.ffmpegProcess) {
367
- stopFFmpegProcess(this.ffmpegProcess);
429
+ this.mixer.stopFFmpegProcess(this.ffmpegProcess);
368
430
  this.ffmpegProcess = null;
369
431
  }
370
432
  this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
@@ -417,7 +479,7 @@ class AudioMixer extends events.EventEmitter {
417
479
  }
418
480
  this.activeSlots[slot] = undefined;
419
481
  }
420
- const { sink, writer, stop } = writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
482
+ const { sink, writer, stop } = this.mixer.writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
421
483
  this.activeSlots[slot] = { sink, writer, stop, trackId: audioTrack.id };
422
484
  (_a = audioTrack.addEventListener) === null || _a === void 0 ? void 0 : _a.call(audioTrack, "ended", () => this.detachParticipant(participantId));
423
485
  }
@@ -436,7 +498,7 @@ class AudioMixer extends events.EventEmitter {
436
498
  this.activeSlots[slot] = undefined;
437
499
  }
438
500
  // Clear any queued audio data for this slot to prevent stale audio
439
- clearSlotQueue(slot);
501
+ this.mixer.clearSlotQueue(slot);
440
502
  this.participantSlots.set(slot, "");
441
503
  }
442
504
  }