@whereby.com/assistant-sdk 0.0.0-canary-20250916072551 → 0.0.0-canary-20250917154617
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +375 -280
- package/dist/index.d.cts +15 -4
- package/dist/index.d.mts +15 -4
- package/dist/index.d.ts +15 -4
- package/dist/index.mjs +374 -281
- package/dist/legacy-esm.js +374 -281
- package/dist/tools.cjs +337 -275
- package/dist/tools.d.ts +1 -0
- package/package.json +7 -6
package/dist/tools.cjs
CHANGED
|
@@ -28,301 +28,362 @@ const STREAM_INPUT_SAMPLE_RATE_IN_HZ = 48000;
|
|
|
28
28
|
const BYTES_PER_SAMPLE = 2;
|
|
29
29
|
// 480 samples per 10ms frame at 48kHz
|
|
30
30
|
const FRAME_10MS_SAMPLES = 480;
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
|
|
45
|
-
}
|
|
46
|
-
({
|
|
47
|
-
enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
48
|
-
enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
49
|
-
wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
50
|
-
wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
51
|
-
lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
52
|
-
});
|
|
53
|
-
let slots = [];
|
|
54
|
-
let stopPacerFn = null;
|
|
55
|
-
let outputPacerState = null;
|
|
56
|
-
/**
|
|
57
|
-
* Simple linear interpolation resampler to convert audio to 48kHz.
|
|
58
|
-
* This handles the common case of 16kHz -> 48kHz (3x upsampling).
|
|
59
|
-
*/
|
|
60
|
-
function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
|
|
61
|
-
const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
|
|
62
|
-
const outputLength = Math.floor(inputFrames * ratio);
|
|
63
|
-
const output = new Int16Array(outputLength);
|
|
64
|
-
for (let i = 0; i < outputLength; i++) {
|
|
65
|
-
const inputIndex = i / ratio;
|
|
66
|
-
const index = Math.floor(inputIndex);
|
|
67
|
-
const fraction = inputIndex - index;
|
|
68
|
-
if (index + 1 < inputSamples.length) {
|
|
69
|
-
const sample1 = inputSamples[index];
|
|
70
|
-
const sample2 = inputSamples[index + 1];
|
|
71
|
-
output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
|
|
72
|
-
}
|
|
73
|
-
else {
|
|
74
|
-
output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
|
|
31
|
+
function createFfmpegMixer() {
|
|
32
|
+
const slotBuffers = new Map();
|
|
33
|
+
function appendAndDrainTo480(slot, newSamples) {
|
|
34
|
+
var _a;
|
|
35
|
+
const prev = (_a = slotBuffers.get(slot)) !== null && _a !== void 0 ? _a : new Int16Array(0);
|
|
36
|
+
const merged = new Int16Array(prev.length + newSamples.length);
|
|
37
|
+
merged.set(prev, 0);
|
|
38
|
+
merged.set(newSamples, prev.length);
|
|
39
|
+
let offset = 0;
|
|
40
|
+
while (merged.length - offset >= FRAME_10MS_SAMPLES) {
|
|
41
|
+
const chunk = merged.subarray(offset, offset + FRAME_10MS_SAMPLES);
|
|
42
|
+
enqueueFrame(slot, chunk); // always 480
|
|
43
|
+
offset += FRAME_10MS_SAMPLES;
|
|
75
44
|
}
|
|
45
|
+
slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
|
|
76
46
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
47
|
+
({
|
|
48
|
+
enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
49
|
+
enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
50
|
+
wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
51
|
+
wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
52
|
+
lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
|
|
53
|
+
});
|
|
54
|
+
let slots = [];
|
|
55
|
+
let stopPacerFn = null;
|
|
56
|
+
let outputPacerState = null;
|
|
57
|
+
/**
|
|
58
|
+
* Simple linear interpolation resampler to convert audio to 48kHz.
|
|
59
|
+
* This handles the common case of 16kHz -> 48kHz (3x upsampling).
|
|
60
|
+
*/
|
|
61
|
+
function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
|
|
62
|
+
const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
|
|
63
|
+
const outputLength = Math.floor(inputFrames * ratio);
|
|
64
|
+
const output = new Int16Array(outputLength);
|
|
65
|
+
for (let i = 0; i < outputLength; i++) {
|
|
66
|
+
const inputIndex = i / ratio;
|
|
67
|
+
const index = Math.floor(inputIndex);
|
|
68
|
+
const fraction = inputIndex - index;
|
|
69
|
+
if (index + 1 < inputSamples.length) {
|
|
70
|
+
const sample1 = inputSamples[index];
|
|
71
|
+
const sample2 = inputSamples[index + 1];
|
|
72
|
+
output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return output;
|
|
85
79
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
*
|
|
94
|
-
* Key behavior:
|
|
95
|
-
* - Writes exactly one frame per period, on a shared wall-clock grid.
|
|
96
|
-
* - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
|
|
97
|
-
* never stalls.
|
|
98
|
-
* - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
|
|
99
|
-
* - Honors Node stream backpressure (`write()` return false) without breaking
|
|
100
|
-
* the timing grid.
|
|
101
|
-
*
|
|
102
|
-
* This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
|
|
103
|
-
* can mix them without slow-downs or drift.
|
|
104
|
-
*
|
|
105
|
-
* Call this once right after spawning FFmpeg:
|
|
106
|
-
* ```ts
|
|
107
|
-
* const ff = spawnFFmpegProcess();
|
|
108
|
-
* startPacer(ff, PARTICIPANT_SLOTS);
|
|
109
|
-
* ```
|
|
110
|
-
*
|
|
111
|
-
* When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
|
|
112
|
-
*
|
|
113
|
-
* @param ff Child process handle from spawn("ffmpeg", ...)
|
|
114
|
-
* @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
|
|
115
|
-
*/
|
|
116
|
-
function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
|
|
117
|
-
if (stopPacerFn) {
|
|
118
|
-
stopPacerFn();
|
|
119
|
-
stopPacerFn = null;
|
|
80
|
+
/**
|
|
81
|
+
* Enqueue an audio frame for paced delivery to the RTCAudioSource.
|
|
82
|
+
*/
|
|
83
|
+
function enqueueOutputFrame(samples) {
|
|
84
|
+
if (outputPacerState) {
|
|
85
|
+
outputPacerState.frameQueue.push(samples);
|
|
86
|
+
}
|
|
120
87
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
88
|
+
/**
|
|
89
|
+
* Start the audio pacer loop for all input slots in an FFmpeg process.
|
|
90
|
+
*
|
|
91
|
+
* The pacer ensures each slot (pipe:3..3+N-1) is written to at a steady
|
|
92
|
+
* real-time rate (e.g. 10 ms = 480 samples @ 48kHz), even if WebRTC frames
|
|
93
|
+
* arrive jittery, bursty, or with slightly different clocks.
|
|
94
|
+
*
|
|
95
|
+
* Key behavior:
|
|
96
|
+
* - Writes exactly one frame per period, on a shared wall-clock grid.
|
|
97
|
+
* - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
|
|
98
|
+
* never stalls.
|
|
99
|
+
* - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
|
|
100
|
+
* - Honors Node stream backpressure (`write()` return false) without breaking
|
|
101
|
+
* the timing grid.
|
|
102
|
+
*
|
|
103
|
+
* This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
|
|
104
|
+
* can mix them without slow-downs or drift.
|
|
105
|
+
*
|
|
106
|
+
* Call this once right after spawning FFmpeg:
|
|
107
|
+
* ```ts
|
|
108
|
+
* const ff = spawnFFmpegProcess();
|
|
109
|
+
* startPacer(ff, PARTICIPANT_SLOTS);
|
|
110
|
+
* ```
|
|
111
|
+
*
|
|
112
|
+
* When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
|
|
113
|
+
*
|
|
114
|
+
* @param ff Child process handle from spawn("ffmpeg", ...)
|
|
115
|
+
* @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
|
|
116
|
+
*/
|
|
117
|
+
function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
|
|
118
|
+
if (stopPacerFn) {
|
|
119
|
+
stopPacerFn();
|
|
120
|
+
stopPacerFn = null;
|
|
121
|
+
}
|
|
122
|
+
const writers = Array.from({ length: slotCount }, (_, i) => ff.stdio[3 + i]);
|
|
123
|
+
const nowMs = () => Number(process.hrtime.bigint()) / 1e6;
|
|
124
|
+
const outputFrameMs = (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms
|
|
125
|
+
const t0 = nowMs();
|
|
126
|
+
slots = Array.from({ length: slotCount }, () => ({
|
|
127
|
+
q: [],
|
|
128
|
+
lastFrames: FRAME_10MS_SAMPLES, // keep constant
|
|
129
|
+
nextDueMs: t0 + (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000,
|
|
130
|
+
}));
|
|
131
|
+
outputPacerState = {
|
|
132
|
+
frameQueue: [],
|
|
133
|
+
nextDueMs: t0 + outputFrameMs,
|
|
134
|
+
rtcAudioSource,
|
|
135
|
+
onAudioStreamReady,
|
|
136
|
+
didEmitReadyEvent: false,
|
|
137
|
+
};
|
|
138
|
+
const iv = setInterval(() => {
|
|
139
|
+
const t = nowMs();
|
|
140
|
+
for (let s = 0; s < slotCount; s++) {
|
|
141
|
+
const st = slots[s];
|
|
142
|
+
const w = writers[s];
|
|
143
|
+
const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms if 480, 20ms if 960
|
|
144
|
+
if (t >= st.nextDueMs) {
|
|
145
|
+
const buf = st.q.length ? st.q.shift() : Buffer.alloc(st.lastFrames * BYTES_PER_SAMPLE);
|
|
146
|
+
if (!w.write(buf)) {
|
|
147
|
+
// Just continue without adding drain listener - backpressure will naturally resolve
|
|
148
|
+
const late = t - st.nextDueMs;
|
|
149
|
+
const steps = Math.max(1, Math.ceil(late / frameMs));
|
|
150
|
+
st.nextDueMs += steps * frameMs;
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
147
153
|
const late = t - st.nextDueMs;
|
|
148
154
|
const steps = Math.max(1, Math.ceil(late / frameMs));
|
|
149
155
|
st.nextDueMs += steps * frameMs;
|
|
150
|
-
continue;
|
|
151
156
|
}
|
|
152
|
-
const late = t - st.nextDueMs;
|
|
153
|
-
const steps = Math.max(1, Math.ceil(late / frameMs));
|
|
154
|
-
st.nextDueMs += steps * frameMs;
|
|
155
157
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
158
|
+
if (!outputPacerState)
|
|
159
|
+
return;
|
|
160
|
+
// Handle output pacer for RTCAudioSource
|
|
161
|
+
const state = outputPacerState;
|
|
162
|
+
if (t >= state.nextDueMs) {
|
|
163
|
+
const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
|
|
164
|
+
if (!state.didEmitReadyEvent) {
|
|
165
|
+
state.onAudioStreamReady();
|
|
166
|
+
state.didEmitReadyEvent = true;
|
|
167
|
+
}
|
|
168
|
+
state.rtcAudioSource.onData({
|
|
169
|
+
samples: samples,
|
|
170
|
+
sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
|
|
171
|
+
});
|
|
172
|
+
const late = t - state.nextDueMs;
|
|
173
|
+
const steps = Math.max(1, Math.ceil(late / outputFrameMs));
|
|
174
|
+
state.nextDueMs += steps * outputFrameMs;
|
|
166
175
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
|
|
170
|
-
});
|
|
171
|
-
const late = t - state.nextDueMs;
|
|
172
|
-
const steps = Math.max(1, Math.ceil(late / outputFrameMs));
|
|
173
|
-
state.nextDueMs += steps * outputFrameMs;
|
|
174
|
-
}
|
|
175
|
-
}, 5);
|
|
176
|
-
stopPacerFn = () => clearInterval(iv);
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Stop the audio pacer loop and clear all input slots.
|
|
180
|
-
* Call this before killing the FFmpeg process to ensure clean shutdown.
|
|
181
|
-
*/
|
|
182
|
-
function stopPacer() {
|
|
183
|
-
if (stopPacerFn)
|
|
184
|
-
stopPacerFn();
|
|
185
|
-
stopPacerFn = null;
|
|
186
|
-
slots = [];
|
|
187
|
-
}
|
|
188
|
-
/**
|
|
189
|
-
* Queue a live frame for a given slot (0..N-1).
|
|
190
|
-
* Auto-resnaps the slot's schedule if the frame size (480/960) changes.
|
|
191
|
-
*/
|
|
192
|
-
function enqueueFrame(slot, samples, numberOfFrames) {
|
|
193
|
-
const st = slots[slot];
|
|
194
|
-
if (!st)
|
|
195
|
-
return;
|
|
196
|
-
const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
|
|
197
|
-
st.q.push(buf);
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Clear the audio queue for a specific slot when a participant leaves.
|
|
201
|
-
* This prevents stale audio data from continuing to play after disconnect.
|
|
202
|
-
*/
|
|
203
|
-
function clearSlotQueue(slot) {
|
|
204
|
-
const st = slots[slot];
|
|
205
|
-
if (st) {
|
|
206
|
-
st.q = [];
|
|
207
|
-
const now = Number(process.hrtime.bigint()) / 1e6;
|
|
208
|
-
const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
|
|
209
|
-
st.nextDueMs = now + frameMs;
|
|
176
|
+
}, 5);
|
|
177
|
+
stopPacerFn = () => clearInterval(iv);
|
|
210
178
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
|
|
179
|
+
/**
|
|
180
|
+
* Stop the audio pacer loop and clear all input slots.
|
|
181
|
+
* Call this before killing the FFmpeg process to ensure clean shutdown.
|
|
182
|
+
*/
|
|
183
|
+
function stopPacer() {
|
|
184
|
+
if (stopPacerFn)
|
|
185
|
+
stopPacerFn();
|
|
186
|
+
stopPacerFn = null;
|
|
187
|
+
slots = [];
|
|
188
|
+
slotBuffers.clear();
|
|
189
|
+
outputPacerState = null;
|
|
223
190
|
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
191
|
+
/**
|
|
192
|
+
* Queue a live frame for a given slot (0..N-1).
|
|
193
|
+
* Auto-resnaps the slot's schedule if the frame size (480/960) changes.
|
|
194
|
+
*/
|
|
195
|
+
function enqueueFrame(slot, samples, numberOfFrames) {
|
|
196
|
+
const st = slots[slot];
|
|
197
|
+
if (!st)
|
|
198
|
+
return;
|
|
199
|
+
const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
|
|
200
|
+
st.q.push(buf);
|
|
227
201
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
* @return The spawned FFmpeg process.
|
|
241
|
-
*/
|
|
242
|
-
function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
|
|
243
|
-
const stdio = ["ignore", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
|
|
244
|
-
const args = getFFmpegArguments();
|
|
245
|
-
const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
|
|
246
|
-
startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
|
|
247
|
-
ffmpegProcess.stderr.setEncoding("utf8");
|
|
248
|
-
ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
|
|
249
|
-
ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
|
|
250
|
-
let audioBuffer = Buffer.alloc(0);
|
|
251
|
-
const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
|
|
252
|
-
ffmpegProcess.stdout.on("data", (chunk) => {
|
|
253
|
-
audioBuffer = Buffer.concat([audioBuffer, chunk]);
|
|
254
|
-
while (audioBuffer.length >= FRAME_SIZE_BYTES) {
|
|
255
|
-
const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
|
|
256
|
-
const samples = new Int16Array(FRAME_10MS_SAMPLES);
|
|
257
|
-
for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
|
|
258
|
-
samples[i] = frameData.readInt16LE(i * 2);
|
|
259
|
-
}
|
|
260
|
-
enqueueOutputFrame(samples);
|
|
261
|
-
audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
|
|
202
|
+
/**
|
|
203
|
+
* Clear the audio queue for a specific slot when a participant leaves.
|
|
204
|
+
* This prevents stale audio data from continuing to play after disconnect.
|
|
205
|
+
*/
|
|
206
|
+
function clearSlotQueue(slot) {
|
|
207
|
+
const st = slots[slot];
|
|
208
|
+
if (st) {
|
|
209
|
+
st.q = [];
|
|
210
|
+
slotBuffers.delete(slot);
|
|
211
|
+
const now = Number(process.hrtime.bigint()) / 1e6;
|
|
212
|
+
const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
|
|
213
|
+
st.nextDueMs = now + frameMs;
|
|
262
214
|
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
*/
|
|
276
|
-
function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
|
|
277
|
-
const writer = ffmpegProcess.stdio[3 + slot];
|
|
278
|
-
const sink = new AudioSink(audioTrack);
|
|
279
|
-
const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
|
|
280
|
-
if (ch !== 1 || bitsPerSample !== 16)
|
|
281
|
-
return;
|
|
282
|
-
let out = samples;
|
|
283
|
-
if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
|
|
284
|
-
const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
|
|
285
|
-
out = resampled;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Get the FFmpeg arguments for debugging, which writes each participant's audio to a separate WAV file
|
|
218
|
+
* and also mixes them into a single WAV file.
|
|
219
|
+
* This is useful for inspecting the audio quality and timing of each participant.
|
|
220
|
+
*/
|
|
221
|
+
function getFFmpegArgumentsDebug() {
|
|
222
|
+
const N = PARTICIPANT_SLOTS;
|
|
223
|
+
const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
|
|
224
|
+
const ffArgs = [];
|
|
225
|
+
for (let i = 0; i < N; i++) {
|
|
226
|
+
ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
|
|
286
227
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
try {
|
|
291
|
-
unsubscribe();
|
|
292
|
-
sink.stop();
|
|
228
|
+
const pre = [];
|
|
229
|
+
for (let i = 0; i < N; i++) {
|
|
230
|
+
pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS,asplit=2[a${i}tap][a${i}mix]`);
|
|
293
231
|
}
|
|
294
|
-
|
|
295
|
-
|
|
232
|
+
const mixInputs = Array.from({ length: N }, (_, i) => `[a${i}mix]`).join("");
|
|
233
|
+
const filter = `${pre.join(";")};${mixInputs}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
|
|
234
|
+
ffArgs.push("-hide_banner", "-nostats", "-loglevel", "info", "-y", "-filter_complex", filter);
|
|
235
|
+
for (let i = 0; i < N; i++) {
|
|
236
|
+
ffArgs.push("-map", `[a${i}tap]`, "-f", "wav", "-c:a", "pcm_s16le", `pre${i}.wav`);
|
|
296
237
|
}
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
}
|
|
300
|
-
/**
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
238
|
+
ffArgs.push("-map", "[mix]", "-f", "wav", "-c:a", "pcm_s16le", "mixed.wav");
|
|
239
|
+
return ffArgs;
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Get the FFmpeg arguments for mixing audio from multiple participants.
|
|
243
|
+
* This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
|
|
244
|
+
* The output is in PCM 16-bit little-endian format at 48kHz sample rate.
|
|
245
|
+
*/
|
|
246
|
+
function getFFmpegArguments() {
|
|
247
|
+
const N = PARTICIPANT_SLOTS;
|
|
248
|
+
const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
|
|
249
|
+
const ffArgs = [];
|
|
250
|
+
for (let i = 0; i < N; i++) {
|
|
251
|
+
ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
|
|
311
252
|
}
|
|
312
|
-
|
|
313
|
-
|
|
253
|
+
const pre = [];
|
|
254
|
+
for (let i = 0; i < N; i++) {
|
|
255
|
+
pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS[a${i}]`);
|
|
314
256
|
}
|
|
315
|
-
|
|
316
|
-
|
|
257
|
+
const labels = Array.from({ length: N }, (_, i) => `[a${i}]`).join("");
|
|
258
|
+
const amix = `${labels}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
|
|
259
|
+
const filter = `${pre.join(";")};${amix}`;
|
|
260
|
+
ffArgs.push("-hide_banner", "-nostats", "-loglevel", "error", "-filter_complex", filter, "-map", "[mix]", "-f", "s16le", "-ar", String(SR), "-ac", "1", "-c:a", "pcm_s16le", "pipe:1");
|
|
261
|
+
return ffArgs;
|
|
262
|
+
}
|
|
263
|
+
/*
|
|
264
|
+
* Spawn a new FFmpeg process for debugging purposes.
|
|
265
|
+
* This will write each participant's audio to a separate WAV file and also mix them into a single WAV file.
|
|
266
|
+
* The output files will be named pre0.wav, pre1.wav, ..., and mixed.wav.
|
|
267
|
+
* The process will log its output to stderr.
|
|
268
|
+
* @return The spawned FFmpeg process.
|
|
269
|
+
*/
|
|
270
|
+
function spawnFFmpegProcessDebug(rtcAudioSource, onAudioStreamReady) {
|
|
271
|
+
const stdio = ["ignore", "ignore", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
|
|
272
|
+
const args = getFFmpegArgumentsDebug();
|
|
273
|
+
const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
|
|
274
|
+
startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
|
|
275
|
+
ffmpegProcess.stderr.setEncoding("utf8");
|
|
276
|
+
ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
|
|
277
|
+
ffmpegProcess.on("error", () => console.error("FFmpeg process error (debug): is ffmpeg installed?"));
|
|
278
|
+
return ffmpegProcess;
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Spawn a new FFmpeg process for mixing audio from multiple participants.
|
|
282
|
+
* This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
|
|
283
|
+
* The output is in PCM 16-bit little-endian format at 48kHz sample rate.
|
|
284
|
+
* The process will log its output to stderr.
|
|
285
|
+
* @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
|
|
286
|
+
* @return The spawned FFmpeg process.
|
|
287
|
+
*/
|
|
288
|
+
function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
|
|
289
|
+
const stdio = ["pipe", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
|
|
290
|
+
const args = getFFmpegArguments();
|
|
291
|
+
const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
|
|
292
|
+
startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
|
|
293
|
+
ffmpegProcess.stderr.setEncoding("utf8");
|
|
294
|
+
ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
|
|
295
|
+
ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
|
|
296
|
+
let audioBuffer = Buffer.alloc(0);
|
|
297
|
+
const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
|
|
298
|
+
ffmpegProcess.stdout.on("data", (chunk) => {
|
|
299
|
+
audioBuffer = Buffer.concat([audioBuffer, chunk]);
|
|
300
|
+
while (audioBuffer.length >= FRAME_SIZE_BYTES) {
|
|
301
|
+
const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
|
|
302
|
+
const samples = new Int16Array(FRAME_10MS_SAMPLES);
|
|
303
|
+
for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
|
|
304
|
+
samples[i] = frameData.readInt16LE(i * 2);
|
|
305
|
+
}
|
|
306
|
+
enqueueOutputFrame(samples);
|
|
307
|
+
audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
return ffmpegProcess;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Write audio data from a MediaStreamTrack to the FFmpeg process.
|
|
314
|
+
* This function creates an AudioSink for the track and sets up a data handler
|
|
315
|
+
* that enqueues audio frames into the pacer.
|
|
316
|
+
*
|
|
317
|
+
* @param ffmpegProcess The FFmpeg process to which audio data will be written.
|
|
318
|
+
* @param slot The participant slot number (0..N-1) to which this track belongs.
|
|
319
|
+
* @param audioTrack The MediaStreamTrack containing the audio data.
|
|
320
|
+
* @return An object containing the AudioSink, the writable stream, and a stop function.
|
|
321
|
+
*/
|
|
322
|
+
function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
|
|
323
|
+
const writer = ffmpegProcess.stdio[3 + slot];
|
|
324
|
+
const sink = new AudioSink(audioTrack);
|
|
325
|
+
const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
|
|
326
|
+
if (ch !== 1 || bitsPerSample !== 16)
|
|
327
|
+
return;
|
|
328
|
+
let out = samples;
|
|
329
|
+
if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
|
|
330
|
+
const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
|
|
331
|
+
out = resampled;
|
|
332
|
+
}
|
|
333
|
+
appendAndDrainTo480(slot, out);
|
|
334
|
+
});
|
|
335
|
+
const stop = () => {
|
|
317
336
|
try {
|
|
318
|
-
|
|
337
|
+
unsubscribe();
|
|
338
|
+
sink.stop();
|
|
319
339
|
}
|
|
320
|
-
catch (
|
|
321
|
-
console.error("Failed to
|
|
340
|
+
catch (_a) {
|
|
341
|
+
console.error("Failed to stop AudioSink");
|
|
342
|
+
}
|
|
343
|
+
};
|
|
344
|
+
return { sink, writer, stop };
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Stop the FFmpeg process and clean up all resources.
|
|
348
|
+
* This function will unpipe the stdout, end all writable streams for each participant slot,
|
|
349
|
+
* and kill the FFmpeg process.
|
|
350
|
+
* @param ffmpegProcess The FFmpeg process to stop.
|
|
351
|
+
*/
|
|
352
|
+
function stopFFmpegProcess(ffmpegProcess) {
|
|
353
|
+
var _a, _b;
|
|
354
|
+
stopPacer();
|
|
355
|
+
if (ffmpegProcess && !ffmpegProcess.killed) {
|
|
356
|
+
try {
|
|
357
|
+
ffmpegProcess.stdout.unpipe();
|
|
358
|
+
}
|
|
359
|
+
catch (_c) {
|
|
360
|
+
console.error("Failed to unpipe ffmpeg stdout");
|
|
361
|
+
}
|
|
362
|
+
for (let i = 0; i < PARTICIPANT_SLOTS; i++) {
|
|
363
|
+
const w = ffmpegProcess.stdio[3 + i];
|
|
364
|
+
try {
|
|
365
|
+
w.end();
|
|
366
|
+
}
|
|
367
|
+
catch (_d) {
|
|
368
|
+
console.error("Failed to end ffmpeg writable stream");
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
try {
|
|
372
|
+
(_a = ffmpegProcess.stdin) === null || _a === void 0 ? void 0 : _a.write("q\n");
|
|
373
|
+
(_b = ffmpegProcess.stdin) === null || _b === void 0 ? void 0 : _b.end();
|
|
374
|
+
}
|
|
375
|
+
catch (_e) {
|
|
376
|
+
console.error("Failed to end ffmpeg stdin");
|
|
322
377
|
}
|
|
323
378
|
}
|
|
324
|
-
ffmpegProcess.kill("SIGTERM");
|
|
325
379
|
}
|
|
380
|
+
return {
|
|
381
|
+
spawnFFmpegProcess,
|
|
382
|
+
spawnFFmpegProcessDebug,
|
|
383
|
+
writeAudioDataToFFmpeg,
|
|
384
|
+
stopFFmpegProcess,
|
|
385
|
+
clearSlotQueue,
|
|
386
|
+
};
|
|
326
387
|
}
|
|
327
388
|
|
|
328
389
|
class AudioMixer extends events.EventEmitter {
|
|
@@ -333,6 +394,7 @@ class AudioMixer extends events.EventEmitter {
|
|
|
333
394
|
this.rtcAudioSource = null;
|
|
334
395
|
this.participantSlots = new Map();
|
|
335
396
|
this.activeSlots = {};
|
|
397
|
+
this.mixer = createFfmpegMixer();
|
|
336
398
|
this.setupMediaStream();
|
|
337
399
|
this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
|
|
338
400
|
this.onStreamReady = onStreamReady;
|
|
@@ -351,7 +413,7 @@ class AudioMixer extends events.EventEmitter {
|
|
|
351
413
|
return;
|
|
352
414
|
}
|
|
353
415
|
if (!this.ffmpegProcess && this.rtcAudioSource) {
|
|
354
|
-
this.ffmpegProcess = spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
|
|
416
|
+
this.ffmpegProcess = this.mixer.spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
|
|
355
417
|
}
|
|
356
418
|
for (const p of participants)
|
|
357
419
|
this.attachParticipantIfNeeded(p);
|
|
@@ -364,7 +426,7 @@ class AudioMixer extends events.EventEmitter {
|
|
|
364
426
|
}
|
|
365
427
|
stopAudioMixer() {
|
|
366
428
|
if (this.ffmpegProcess) {
|
|
367
|
-
stopFFmpegProcess(this.ffmpegProcess);
|
|
429
|
+
this.mixer.stopFFmpegProcess(this.ffmpegProcess);
|
|
368
430
|
this.ffmpegProcess = null;
|
|
369
431
|
}
|
|
370
432
|
this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
|
|
@@ -417,7 +479,7 @@ class AudioMixer extends events.EventEmitter {
|
|
|
417
479
|
}
|
|
418
480
|
this.activeSlots[slot] = undefined;
|
|
419
481
|
}
|
|
420
|
-
const { sink, writer, stop } = writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
|
|
482
|
+
const { sink, writer, stop } = this.mixer.writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
|
|
421
483
|
this.activeSlots[slot] = { sink, writer, stop, trackId: audioTrack.id };
|
|
422
484
|
(_a = audioTrack.addEventListener) === null || _a === void 0 ? void 0 : _a.call(audioTrack, "ended", () => this.detachParticipant(participantId));
|
|
423
485
|
}
|
|
@@ -436,7 +498,7 @@ class AudioMixer extends events.EventEmitter {
|
|
|
436
498
|
this.activeSlots[slot] = undefined;
|
|
437
499
|
}
|
|
438
500
|
// Clear any queued audio data for this slot to prevent stale audio
|
|
439
|
-
clearSlotQueue(slot);
|
|
501
|
+
this.mixer.clearSlotQueue(slot);
|
|
440
502
|
this.participantSlots.set(slot, "");
|
|
441
503
|
}
|
|
442
504
|
}
|