@whereby.com/assistant-sdk 0.0.0-canary-20250916140846 → 0.0.0-canary-20250917154617

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -78,301 +78,362 @@ const STREAM_INPUT_SAMPLE_RATE_IN_HZ = 48000;
78
78
  const BYTES_PER_SAMPLE = 2;
79
79
  // 480 samples per 10ms frame at 48kHz
80
80
  const FRAME_10MS_SAMPLES = 480;
81
- const slotBuffers = new Map();
82
- function appendAndDrainTo480(slot, newSamples) {
83
- var _a;
84
- const prev = (_a = slotBuffers.get(slot)) !== null && _a !== void 0 ? _a : new Int16Array(0);
85
- const merged = new Int16Array(prev.length + newSamples.length);
86
- merged.set(prev, 0);
87
- merged.set(newSamples, prev.length);
88
- let offset = 0;
89
- while (merged.length - offset >= FRAME_10MS_SAMPLES) {
90
- const chunk = merged.subarray(offset, offset + FRAME_10MS_SAMPLES);
91
- enqueueFrame(slot, chunk); // always 480
92
- offset += FRAME_10MS_SAMPLES;
93
- }
94
- slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
95
- }
96
- ({
97
- enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
98
- enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
99
- wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
100
- wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
101
- lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
102
- });
103
- let slots = [];
104
- let stopPacerFn = null;
105
- let outputPacerState = null;
106
- /**
107
- * Simple linear interpolation resampler to convert audio to 48kHz.
108
- * This handles the common case of 16kHz -> 48kHz (3x upsampling).
109
- */
110
- function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
111
- const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
112
- const outputLength = Math.floor(inputFrames * ratio);
113
- const output = new Int16Array(outputLength);
114
- for (let i = 0; i < outputLength; i++) {
115
- const inputIndex = i / ratio;
116
- const index = Math.floor(inputIndex);
117
- const fraction = inputIndex - index;
118
- if (index + 1 < inputSamples.length) {
119
- const sample1 = inputSamples[index];
120
- const sample2 = inputSamples[index + 1];
121
- output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
81
+ function createFfmpegMixer() {
82
+ const slotBuffers = new Map();
83
+ function appendAndDrainTo480(slot, newSamples) {
84
+ var _a;
85
+ const prev = (_a = slotBuffers.get(slot)) !== null && _a !== void 0 ? _a : new Int16Array(0);
86
+ const merged = new Int16Array(prev.length + newSamples.length);
87
+ merged.set(prev, 0);
88
+ merged.set(newSamples, prev.length);
89
+ let offset = 0;
90
+ while (merged.length - offset >= FRAME_10MS_SAMPLES) {
91
+ const chunk = merged.subarray(offset, offset + FRAME_10MS_SAMPLES);
92
+ enqueueFrame(slot, chunk); // always 480
93
+ offset += FRAME_10MS_SAMPLES;
122
94
  }
123
- else {
124
- output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
95
+ slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
96
+ }
97
+ ({
98
+ enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
99
+ enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
100
+ wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
101
+ wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
102
+ lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
103
+ });
104
+ let slots = [];
105
+ let stopPacerFn = null;
106
+ let outputPacerState = null;
107
+ /**
108
+ * Simple linear interpolation resampler to convert audio to 48kHz.
109
+ * This handles the common case of 16kHz -> 48kHz (3x upsampling).
110
+ */
111
+ function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
112
+ const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
113
+ const outputLength = Math.floor(inputFrames * ratio);
114
+ const output = new Int16Array(outputLength);
115
+ for (let i = 0; i < outputLength; i++) {
116
+ const inputIndex = i / ratio;
117
+ const index = Math.floor(inputIndex);
118
+ const fraction = inputIndex - index;
119
+ if (index + 1 < inputSamples.length) {
120
+ const sample1 = inputSamples[index];
121
+ const sample2 = inputSamples[index + 1];
122
+ output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
123
+ }
124
+ else {
125
+ output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
126
+ }
127
+ }
128
+ return output;
129
+ }
130
+ /**
131
+ * Enqueue an audio frame for paced delivery to the RTCAudioSource.
132
+ */
133
+ function enqueueOutputFrame(samples) {
134
+ if (outputPacerState) {
135
+ outputPacerState.frameQueue.push(samples);
125
136
  }
126
137
  }
127
- return output;
128
- }
129
- /**
130
- * Enqueue an audio frame for paced delivery to the RTCAudioSource.
131
- */
132
- function enqueueOutputFrame(samples) {
133
- if (outputPacerState) {
134
- outputPacerState.frameQueue.push(samples);
135
- }
136
- }
137
- /**
138
- * Start the audio pacer loop for all input slots in an FFmpeg process.
139
- *
140
- * The pacer ensures each slot (pipe:3..3+N-1) is written to at a steady
141
- * real-time rate (e.g. 10 ms = 480 samples @ 48kHz), even if WebRTC frames
142
- * arrive jittery, bursty, or with slightly different clocks.
143
- *
144
- * Key behavior:
145
- * - Writes exactly one frame per period, on a shared wall-clock grid.
146
- * - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
147
- * never stalls.
148
- * - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
149
- * - Honors Node stream backpressure (`write()` return false) without breaking
150
- * the timing grid.
151
- *
152
- * This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
153
- * can mix them without slow-downs or drift.
154
- *
155
- * Call this once right after spawning FFmpeg:
156
- * ```ts
157
- * const ff = spawnFFmpegProcess();
158
- * startPacer(ff, PARTICIPANT_SLOTS);
159
- * ```
160
- *
161
- * When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
162
- *
163
- * @param ff Child process handle from spawn("ffmpeg", ...)
164
- * @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
165
- */
166
- function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
167
- if (stopPacerFn) {
168
- stopPacerFn();
169
- stopPacerFn = null;
170
- }
171
- const writers = Array.from({ length: slotCount }, (_, i) => ff.stdio[3 + i]);
172
- const nowMs = () => Number(process.hrtime.bigint()) / 1e6;
173
- const outputFrameMs = (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms
174
- const t0 = nowMs();
175
- slots = Array.from({ length: slotCount }, () => ({
176
- q: [],
177
- lastFrames: FRAME_10MS_SAMPLES, // keep constant
178
- nextDueMs: t0 + (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000,
179
- }));
180
- outputPacerState = {
181
- frameQueue: [],
182
- nextDueMs: t0 + outputFrameMs,
183
- rtcAudioSource,
184
- onAudioStreamReady,
185
- didEmitReadyEvent: false,
186
- };
187
- const iv = setInterval(() => {
188
- const t = nowMs();
189
- for (let s = 0; s < slotCount; s++) {
190
- const st = slots[s];
191
- const w = writers[s];
192
- const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms if 480, 20ms if 960
193
- if (t >= st.nextDueMs) {
194
- const buf = st.q.length ? st.q.shift() : Buffer.alloc(st.lastFrames * BYTES_PER_SAMPLE);
195
- if (!w.write(buf)) {
196
- // Just continue without adding drain listener - backpressure will naturally resolve
138
+ /**
139
+ * Start the audio pacer loop for all input slots in an FFmpeg process.
140
+ *
141
+ * The pacer ensures each slot (pipe:3..3+N-1) is written to at a steady
142
+ * real-time rate (e.g. 10 ms = 480 samples @ 48kHz), even if WebRTC frames
143
+ * arrive jittery, bursty, or with slightly different clocks.
144
+ *
145
+ * Key behavior:
146
+ * - Writes exactly one frame per period, on a shared wall-clock grid.
147
+ * - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
148
+ * never stalls.
149
+ * - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
150
+ * - Honors Node stream backpressure (`write()` return false) without breaking
151
+ * the timing grid.
152
+ *
153
+ * This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
154
+ * can mix them without slow-downs or drift.
155
+ *
156
+ * Call this once right after spawning FFmpeg:
157
+ * ```ts
158
+ * const ff = spawnFFmpegProcess();
159
+ * startPacer(ff, PARTICIPANT_SLOTS);
160
+ * ```
161
+ *
162
+ * When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
163
+ *
164
+ * @param ff Child process handle from spawn("ffmpeg", ...)
165
+ * @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
166
+ */
167
+ function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
168
+ if (stopPacerFn) {
169
+ stopPacerFn();
170
+ stopPacerFn = null;
171
+ }
172
+ const writers = Array.from({ length: slotCount }, (_, i) => ff.stdio[3 + i]);
173
+ const nowMs = () => Number(process.hrtime.bigint()) / 1e6;
174
+ const outputFrameMs = (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms
175
+ const t0 = nowMs();
176
+ slots = Array.from({ length: slotCount }, () => ({
177
+ q: [],
178
+ lastFrames: FRAME_10MS_SAMPLES, // keep constant
179
+ nextDueMs: t0 + (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000,
180
+ }));
181
+ outputPacerState = {
182
+ frameQueue: [],
183
+ nextDueMs: t0 + outputFrameMs,
184
+ rtcAudioSource,
185
+ onAudioStreamReady,
186
+ didEmitReadyEvent: false,
187
+ };
188
+ const iv = setInterval(() => {
189
+ const t = nowMs();
190
+ for (let s = 0; s < slotCount; s++) {
191
+ const st = slots[s];
192
+ const w = writers[s];
193
+ const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms if 480, 20ms if 960
194
+ if (t >= st.nextDueMs) {
195
+ const buf = st.q.length ? st.q.shift() : Buffer.alloc(st.lastFrames * BYTES_PER_SAMPLE);
196
+ if (!w.write(buf)) {
197
+ // Just continue without adding drain listener - backpressure will naturally resolve
198
+ const late = t - st.nextDueMs;
199
+ const steps = Math.max(1, Math.ceil(late / frameMs));
200
+ st.nextDueMs += steps * frameMs;
201
+ continue;
202
+ }
197
203
  const late = t - st.nextDueMs;
198
204
  const steps = Math.max(1, Math.ceil(late / frameMs));
199
205
  st.nextDueMs += steps * frameMs;
200
- continue;
201
206
  }
202
- const late = t - st.nextDueMs;
203
- const steps = Math.max(1, Math.ceil(late / frameMs));
204
- st.nextDueMs += steps * frameMs;
205
207
  }
206
- }
207
- if (!outputPacerState)
208
- return;
209
- // Handle output pacer for RTCAudioSource
210
- const state = outputPacerState;
211
- if (t >= state.nextDueMs) {
212
- const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
213
- if (!state.didEmitReadyEvent) {
214
- state.onAudioStreamReady();
215
- state.didEmitReadyEvent = true;
208
+ if (!outputPacerState)
209
+ return;
210
+ // Handle output pacer for RTCAudioSource
211
+ const state = outputPacerState;
212
+ if (t >= state.nextDueMs) {
213
+ const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
214
+ if (!state.didEmitReadyEvent) {
215
+ state.onAudioStreamReady();
216
+ state.didEmitReadyEvent = true;
217
+ }
218
+ state.rtcAudioSource.onData({
219
+ samples: samples,
220
+ sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
221
+ });
222
+ const late = t - state.nextDueMs;
223
+ const steps = Math.max(1, Math.ceil(late / outputFrameMs));
224
+ state.nextDueMs += steps * outputFrameMs;
216
225
  }
217
- state.rtcAudioSource.onData({
218
- samples: samples,
219
- sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
220
- });
221
- const late = t - state.nextDueMs;
222
- const steps = Math.max(1, Math.ceil(late / outputFrameMs));
223
- state.nextDueMs += steps * outputFrameMs;
226
+ }, 5);
227
+ stopPacerFn = () => clearInterval(iv);
228
+ }
229
+ /**
230
+ * Stop the audio pacer loop and clear all input slots.
231
+ * Call this before killing the FFmpeg process to ensure clean shutdown.
232
+ */
233
+ function stopPacer() {
234
+ if (stopPacerFn)
235
+ stopPacerFn();
236
+ stopPacerFn = null;
237
+ slots = [];
238
+ slotBuffers.clear();
239
+ outputPacerState = null;
240
+ }
241
+ /**
242
+ * Queue a live frame for a given slot (0..N-1).
243
+ * Auto-resnaps the slot's schedule if the frame size (480/960) changes.
244
+ */
245
+ function enqueueFrame(slot, samples, numberOfFrames) {
246
+ const st = slots[slot];
247
+ if (!st)
248
+ return;
249
+ const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
250
+ st.q.push(buf);
251
+ }
252
+ /**
253
+ * Clear the audio queue for a specific slot when a participant leaves.
254
+ * This prevents stale audio data from continuing to play after disconnect.
255
+ */
256
+ function clearSlotQueue(slot) {
257
+ const st = slots[slot];
258
+ if (st) {
259
+ st.q = [];
260
+ slotBuffers.delete(slot);
261
+ const now = Number(process.hrtime.bigint()) / 1e6;
262
+ const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
263
+ st.nextDueMs = now + frameMs;
224
264
  }
225
- }, 5);
226
- stopPacerFn = () => clearInterval(iv);
227
- }
228
- /**
229
- * Stop the audio pacer loop and clear all input slots.
230
- * Call this before killing the FFmpeg process to ensure clean shutdown.
231
- */
232
- function stopPacer() {
233
- if (stopPacerFn)
234
- stopPacerFn();
235
- stopPacerFn = null;
236
- slots = [];
237
- }
238
- /**
239
- * Queue a live frame for a given slot (0..N-1).
240
- * Auto-resnaps the slot's schedule if the frame size (480/960) changes.
241
- */
242
- function enqueueFrame(slot, samples, numberOfFrames) {
243
- const st = slots[slot];
244
- if (!st)
245
- return;
246
- const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
247
- st.q.push(buf);
248
- }
249
- /**
250
- * Clear the audio queue for a specific slot when a participant leaves.
251
- * This prevents stale audio data from continuing to play after disconnect.
252
- */
253
- function clearSlotQueue(slot) {
254
- const st = slots[slot];
255
- if (st) {
256
- st.q = [];
257
- const now = Number(process.hrtime.bigint()) / 1e6;
258
- const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
259
- st.nextDueMs = now + frameMs;
260
265
  }
261
- }
262
- /**
263
- * Get the FFmpeg arguments for mixing audio from multiple participants.
264
- * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
265
- * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
266
- */
267
- function getFFmpegArguments() {
268
- const N = PARTICIPANT_SLOTS;
269
- const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
270
- const ffArgs = [];
271
- for (let i = 0; i < N; i++) {
272
- ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
273
- }
274
- const pre = [];
275
- for (let i = 0; i < N; i++) {
276
- pre.push(`[${i}:a]aresample=async=1:first_pts=0,asetpts=N/SR/TB[a${i}]`);
277
- }
278
- const labels = Array.from({ length: N }, (_, i) => `[a${i}]`).join("");
279
- const amix = `${labels}amix=inputs=${N}:duration=longest:dropout_transition=250:normalize=0[mix]`;
280
- const filter = `${pre.join(";")};${amix}`;
281
- ffArgs.push("-hide_banner", "-nostats", "-loglevel", "error", "-filter_complex", filter, "-map", "[mix]", "-f", "s16le", "-ar", String(SR), "-ac", "1", "-c:a", "pcm_s16le", "pipe:1");
282
- return ffArgs;
283
- }
284
- /**
285
- * Spawn a new FFmpeg process for mixing audio from multiple participants.
286
- * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
287
- * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
288
- * The process will log its output to stderr.
289
- * @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
290
- * @return The spawned FFmpeg process.
291
- */
292
- function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
293
- const stdio = ["ignore", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
294
- const args = getFFmpegArguments();
295
- const ffmpegProcess = spawn("ffmpeg", args, { stdio });
296
- startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
297
- ffmpegProcess.stderr.setEncoding("utf8");
298
- ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
299
- ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
300
- let audioBuffer = Buffer.alloc(0);
301
- const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
302
- ffmpegProcess.stdout.on("data", (chunk) => {
303
- audioBuffer = Buffer.concat([audioBuffer, chunk]);
304
- while (audioBuffer.length >= FRAME_SIZE_BYTES) {
305
- const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
306
- const samples = new Int16Array(FRAME_10MS_SAMPLES);
307
- for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
308
- samples[i] = frameData.readInt16LE(i * 2);
309
- }
310
- enqueueOutputFrame(samples);
311
- audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
266
+ /**
267
+ * Get the FFmpeg arguments for debugging, which writes each participant's audio to a separate WAV file
268
+ * and also mixes them into a single WAV file.
269
+ * This is useful for inspecting the audio quality and timing of each participant.
270
+ */
271
+ function getFFmpegArgumentsDebug() {
272
+ const N = PARTICIPANT_SLOTS;
273
+ const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
274
+ const ffArgs = [];
275
+ for (let i = 0; i < N; i++) {
276
+ ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
312
277
  }
313
- });
314
- return ffmpegProcess;
315
- }
316
- /**
317
- * Write audio data from a MediaStreamTrack to the FFmpeg process.
318
- * This function creates an AudioSink for the track and sets up a data handler
319
- * that enqueues audio frames into the pacer.
320
- *
321
- * @param ffmpegProcess The FFmpeg process to which audio data will be written.
322
- * @param slot The participant slot number (0..N-1) to which this track belongs.
323
- * @param audioTrack The MediaStreamTrack containing the audio data.
324
- * @return An object containing the AudioSink, the writable stream, and a stop function.
325
- */
326
- function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
327
- const writer = ffmpegProcess.stdio[3 + slot];
328
- const sink = new AudioSink(audioTrack);
329
- const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
330
- if (ch !== 1 || bitsPerSample !== 16)
331
- return;
332
- let out = samples;
333
- if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
334
- const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
335
- out = resampled;
278
+ const pre = [];
279
+ for (let i = 0; i < N; i++) {
280
+ pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS,asplit=2[a${i}tap][a${i}mix]`);
336
281
  }
337
- appendAndDrainTo480(slot, out);
338
- });
339
- const stop = () => {
340
- try {
341
- unsubscribe();
342
- sink.stop();
282
+ const mixInputs = Array.from({ length: N }, (_, i) => `[a${i}mix]`).join("");
283
+ const filter = `${pre.join(";")};${mixInputs}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
284
+ ffArgs.push("-hide_banner", "-nostats", "-loglevel", "info", "-y", "-filter_complex", filter);
285
+ for (let i = 0; i < N; i++) {
286
+ ffArgs.push("-map", `[a${i}tap]`, "-f", "wav", "-c:a", "pcm_s16le", `pre${i}.wav`);
343
287
  }
344
- catch (_a) {
345
- console.error("Failed to stop AudioSink");
288
+ ffArgs.push("-map", "[mix]", "-f", "wav", "-c:a", "pcm_s16le", "mixed.wav");
289
+ return ffArgs;
290
+ }
291
+ /**
292
+ * Get the FFmpeg arguments for mixing audio from multiple participants.
293
+ * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
294
+ * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
295
+ */
296
+ function getFFmpegArguments() {
297
+ const N = PARTICIPANT_SLOTS;
298
+ const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
299
+ const ffArgs = [];
300
+ for (let i = 0; i < N; i++) {
301
+ ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
346
302
  }
347
- };
348
- return { sink, writer, stop };
349
- }
350
- /**
351
- * Stop the FFmpeg process and clean up all resources.
352
- * This function will unpipe the stdout, end all writable streams for each participant slot,
353
- * and kill the FFmpeg process.
354
- * @param ffmpegProcess The FFmpeg process to stop.
355
- */
356
- function stopFFmpegProcess(ffmpegProcess) {
357
- stopPacer();
358
- if (ffmpegProcess && !ffmpegProcess.killed) {
359
- try {
360
- ffmpegProcess.stdout.unpipe();
361
- }
362
- catch (_a) {
363
- console.error("Failed to unpipe ffmpeg stdout");
303
+ const pre = [];
304
+ for (let i = 0; i < N; i++) {
305
+ pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS[a${i}]`);
364
306
  }
365
- for (let i = 0; i < PARTICIPANT_SLOTS; i++) {
366
- const w = ffmpegProcess.stdio[3 + i];
307
+ const labels = Array.from({ length: N }, (_, i) => `[a${i}]`).join("");
308
+ const amix = `${labels}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
309
+ const filter = `${pre.join(";")};${amix}`;
310
+ ffArgs.push("-hide_banner", "-nostats", "-loglevel", "error", "-filter_complex", filter, "-map", "[mix]", "-f", "s16le", "-ar", String(SR), "-ac", "1", "-c:a", "pcm_s16le", "pipe:1");
311
+ return ffArgs;
312
+ }
313
+ /*
314
+ * Spawn a new FFmpeg process for debugging purposes.
315
+ * This will write each participant's audio to a separate WAV file and also mix them into a single WAV file.
316
+ * The output files will be named pre0.wav, pre1.wav, ..., and mixed.wav.
317
+ * The process will log its output to stderr.
318
+ * @return The spawned FFmpeg process.
319
+ */
320
+ function spawnFFmpegProcessDebug(rtcAudioSource, onAudioStreamReady) {
321
+ const stdio = ["ignore", "ignore", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
322
+ const args = getFFmpegArgumentsDebug();
323
+ const ffmpegProcess = spawn("ffmpeg", args, { stdio });
324
+ startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
325
+ ffmpegProcess.stderr.setEncoding("utf8");
326
+ ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
327
+ ffmpegProcess.on("error", () => console.error("FFmpeg process error (debug): is ffmpeg installed?"));
328
+ return ffmpegProcess;
329
+ }
330
+ /**
331
+ * Spawn a new FFmpeg process for mixing audio from multiple participants.
332
+ * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
333
+ * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
334
+ * The process will log its output to stderr.
335
+ * @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
336
+ * @return The spawned FFmpeg process.
337
+ */
338
+ function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
339
+ const stdio = ["pipe", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
340
+ const args = getFFmpegArguments();
341
+ const ffmpegProcess = spawn("ffmpeg", args, { stdio });
342
+ startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
343
+ ffmpegProcess.stderr.setEncoding("utf8");
344
+ ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
345
+ ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
346
+ let audioBuffer = Buffer.alloc(0);
347
+ const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
348
+ ffmpegProcess.stdout.on("data", (chunk) => {
349
+ audioBuffer = Buffer.concat([audioBuffer, chunk]);
350
+ while (audioBuffer.length >= FRAME_SIZE_BYTES) {
351
+ const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
352
+ const samples = new Int16Array(FRAME_10MS_SAMPLES);
353
+ for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
354
+ samples[i] = frameData.readInt16LE(i * 2);
355
+ }
356
+ enqueueOutputFrame(samples);
357
+ audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
358
+ }
359
+ });
360
+ return ffmpegProcess;
361
+ }
362
+ /**
363
+ * Write audio data from a MediaStreamTrack to the FFmpeg process.
364
+ * This function creates an AudioSink for the track and sets up a data handler
365
+ * that enqueues audio frames into the pacer.
366
+ *
367
+ * @param ffmpegProcess The FFmpeg process to which audio data will be written.
368
+ * @param slot The participant slot number (0..N-1) to which this track belongs.
369
+ * @param audioTrack The MediaStreamTrack containing the audio data.
370
+ * @return An object containing the AudioSink, the writable stream, and a stop function.
371
+ */
372
+ function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
373
+ const writer = ffmpegProcess.stdio[3 + slot];
374
+ const sink = new AudioSink(audioTrack);
375
+ const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
376
+ if (ch !== 1 || bitsPerSample !== 16)
377
+ return;
378
+ let out = samples;
379
+ if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
380
+ const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
381
+ out = resampled;
382
+ }
383
+ appendAndDrainTo480(slot, out);
384
+ });
385
+ const stop = () => {
386
+ try {
387
+ unsubscribe();
388
+ sink.stop();
389
+ }
390
+ catch (_a) {
391
+ console.error("Failed to stop AudioSink");
392
+ }
393
+ };
394
+ return { sink, writer, stop };
395
+ }
396
+ /**
397
+ * Stop the FFmpeg process and clean up all resources.
398
+ * This function will unpipe the stdout, end all writable streams for each participant slot,
399
+ * and kill the FFmpeg process.
400
+ * @param ffmpegProcess The FFmpeg process to stop.
401
+ */
402
+ function stopFFmpegProcess(ffmpegProcess) {
403
+ var _a, _b;
404
+ stopPacer();
405
+ if (ffmpegProcess && !ffmpegProcess.killed) {
406
+ try {
407
+ ffmpegProcess.stdout.unpipe();
408
+ }
409
+ catch (_c) {
410
+ console.error("Failed to unpipe ffmpeg stdout");
411
+ }
412
+ for (let i = 0; i < PARTICIPANT_SLOTS; i++) {
413
+ const w = ffmpegProcess.stdio[3 + i];
414
+ try {
415
+ w.end();
416
+ }
417
+ catch (_d) {
418
+ console.error("Failed to end ffmpeg writable stream");
419
+ }
420
+ }
367
421
  try {
368
- w.end();
422
+ (_a = ffmpegProcess.stdin) === null || _a === void 0 ? void 0 : _a.write("q\n");
423
+ (_b = ffmpegProcess.stdin) === null || _b === void 0 ? void 0 : _b.end();
369
424
  }
370
- catch (_b) {
371
- console.error("Failed to end ffmpeg writable stream");
425
+ catch (_e) {
426
+ console.error("Failed to end ffmpeg stdin");
372
427
  }
373
428
  }
374
- ffmpegProcess.kill("SIGTERM");
375
429
  }
430
+ return {
431
+ spawnFFmpegProcess,
432
+ spawnFFmpegProcessDebug,
433
+ writeAudioDataToFFmpeg,
434
+ stopFFmpegProcess,
435
+ clearSlotQueue,
436
+ };
376
437
  }
377
438
 
378
439
  class AudioMixer extends EventEmitter {
@@ -383,6 +444,7 @@ class AudioMixer extends EventEmitter {
383
444
  this.rtcAudioSource = null;
384
445
  this.participantSlots = new Map();
385
446
  this.activeSlots = {};
447
+ this.mixer = createFfmpegMixer();
386
448
  this.setupMediaStream();
387
449
  this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
388
450
  this.onStreamReady = onStreamReady;
@@ -401,7 +463,7 @@ class AudioMixer extends EventEmitter {
401
463
  return;
402
464
  }
403
465
  if (!this.ffmpegProcess && this.rtcAudioSource) {
404
- this.ffmpegProcess = spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
466
+ this.ffmpegProcess = this.mixer.spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
405
467
  }
406
468
  for (const p of participants)
407
469
  this.attachParticipantIfNeeded(p);
@@ -414,7 +476,7 @@ class AudioMixer extends EventEmitter {
414
476
  }
415
477
  stopAudioMixer() {
416
478
  if (this.ffmpegProcess) {
417
- stopFFmpegProcess(this.ffmpegProcess);
479
+ this.mixer.stopFFmpegProcess(this.ffmpegProcess);
418
480
  this.ffmpegProcess = null;
419
481
  }
420
482
  this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
@@ -467,7 +529,7 @@ class AudioMixer extends EventEmitter {
467
529
  }
468
530
  this.activeSlots[slot] = undefined;
469
531
  }
470
- const { sink, writer, stop } = writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
532
+ const { sink, writer, stop } = this.mixer.writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
471
533
  this.activeSlots[slot] = { sink, writer, stop, trackId: audioTrack.id };
472
534
  (_a = audioTrack.addEventListener) === null || _a === void 0 ? void 0 : _a.call(audioTrack, "ended", () => this.detachParticipant(participantId));
473
535
  }
@@ -486,7 +548,7 @@ class AudioMixer extends EventEmitter {
486
548
  this.activeSlots[slot] = undefined;
487
549
  }
488
550
  // Clear any queued audio data for this slot to prevent stale audio
489
- clearSlotQueue(slot);
551
+ this.mixer.clearSlotQueue(slot);
490
552
  this.participantSlots.set(slot, "");
491
553
  }
492
554
  }