@whereby.com/assistant-sdk 0.0.0-canary-20250916140846 → 0.0.0-canary-20250923130059

package/dist/index.cjs

@@ -35,6 +35,10 @@ const TRIGGER_EVENT_SUCCESS = "trigger_event_success";
 const AUDIO_STREAM_READY = "AUDIO_STREAM_READY";
 const ASSISTANT_JOINED_ROOM = "ASSISTANT_JOINED_ROOM";
 const ASSISTANT_LEFT_ROOM = "ASSISTANT_LEFT_ROOM";
+const PARTICIPANT_VIDEO_TRACK_ADDED = "PARTICIPANT_VIDEO_TRACK_ADDED";
+const PARTICIPANT_VIDEO_TRACK_REMOVED = "PARTICIPANT_VIDEO_TRACK_REMOVED";
+const PARTICIPANT_AUDIO_TRACK_ADDED = "PARTICIPANT_AUDIO_TRACK_ADDED";
+const PARTICIPANT_AUDIO_TRACK_REMOVED = "PARTICIPANT_AUDIO_TRACK_REMOVED";
 
 /******************************************************************************
 Copyright (c) Microsoft Corporation.
@@ -99,301 +103,362 @@ const STREAM_INPUT_SAMPLE_RATE_IN_HZ = 48000;
 const BYTES_PER_SAMPLE = 2;
 // 480 samples per 10ms frame at 48kHz
 const FRAME_10MS_SAMPLES = 480;
-const slotBuffers = new Map();
-function appendAndDrainTo480(slot, newSamples) {
-    var _a;
-    const prev = (_a = slotBuffers.get(slot)) !== null && _a !== void 0 ? _a : new Int16Array(0);
-    const merged = new Int16Array(prev.length + newSamples.length);
-    merged.set(prev, 0);
-    merged.set(newSamples, prev.length);
-    let offset = 0;
-    while (merged.length - offset >= FRAME_10MS_SAMPLES) {
-        const chunk = merged.subarray(offset, offset + FRAME_10MS_SAMPLES);
-        enqueueFrame(slot, chunk); // always 480
-        offset += FRAME_10MS_SAMPLES;
-    }
-    slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
-}
-({
-    enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
-    enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
-    wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
-    wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
-    lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
-});
-let slots = [];
-let stopPacerFn = null;
-let outputPacerState = null;
-/**
- * Simple linear interpolation resampler to convert audio to 48kHz.
- * This handles the common case of 16kHz -> 48kHz (3x upsampling).
- */
-function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
-    const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
-    const outputLength = Math.floor(inputFrames * ratio);
-    const output = new Int16Array(outputLength);
-    for (let i = 0; i < outputLength; i++) {
-        const inputIndex = i / ratio;
-        const index = Math.floor(inputIndex);
-        const fraction = inputIndex - index;
-        if (index + 1 < inputSamples.length) {
-            const sample1 = inputSamples[index];
-            const sample2 = inputSamples[index + 1];
-            output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
+function createFfmpegMixer() {
+    const slotBuffers = new Map();
+    function appendAndDrainTo480(slot, newSamples) {
+        var _a;
+        const prev = (_a = slotBuffers.get(slot)) !== null && _a !== void 0 ? _a : new Int16Array(0);
+        const merged = new Int16Array(prev.length + newSamples.length);
+        merged.set(prev, 0);
+        merged.set(newSamples, prev.length);
+        let offset = 0;
+        while (merged.length - offset >= FRAME_10MS_SAMPLES) {
+            const chunk = merged.subarray(offset, offset + FRAME_10MS_SAMPLES);
+            enqueueFrame(slot, chunk); // always 480
+            offset += FRAME_10MS_SAMPLES;
         }
-        else {
-            output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
+        slotBuffers.set(slot, merged.subarray(offset)); // keep remainder
+    }
+    ({
+        enqFrames: new Array(PARTICIPANT_SLOTS).fill(0),
+        enqSamples: new Array(PARTICIPANT_SLOTS).fill(0),
+        wroteFrames: new Array(PARTICIPANT_SLOTS).fill(0),
+        wroteSamples: new Array(PARTICIPANT_SLOTS).fill(0),
+        lastFramesSeen: new Array(PARTICIPANT_SLOTS).fill(0),
+    });
+    let slots = [];
+    let stopPacerFn = null;
+    let outputPacerState = null;
+    /**
+     * Simple linear interpolation resampler to convert audio to 48kHz.
+     * This handles the common case of 16kHz -> 48kHz (3x upsampling).
+     */
+    function resampleTo48kHz(inputSamples, inputSampleRate, inputFrames) {
+        const ratio = STREAM_INPUT_SAMPLE_RATE_IN_HZ / inputSampleRate;
+        const outputLength = Math.floor(inputFrames * ratio);
+        const output = new Int16Array(outputLength);
+        for (let i = 0; i < outputLength; i++) {
+            const inputIndex = i / ratio;
+            const index = Math.floor(inputIndex);
+            const fraction = inputIndex - index;
+            if (index + 1 < inputSamples.length) {
+                const sample1 = inputSamples[index];
+                const sample2 = inputSamples[index + 1];
+                output[i] = Math.round(sample1 + (sample2 - sample1) * fraction);
+            }
+            else {
+                output[i] = inputSamples[Math.min(index, inputSamples.length - 1)];
+            }
+        }
+        return output;
+    }
+    /**
+     * Enqueue an audio frame for paced delivery to the RTCAudioSource.
+     */
+    function enqueueOutputFrame(samples) {
+        if (outputPacerState) {
+            outputPacerState.frameQueue.push(samples);
         }
     }
-    return output;
-}
-/**
- * Enqueue an audio frame for paced delivery to the RTCAudioSource.
- */
-function enqueueOutputFrame(samples) {
-    if (outputPacerState) {
-        outputPacerState.frameQueue.push(samples);
-    }
-}
-/**
- * Start the audio pacer loop for all input slots in an FFmpeg process.
- *
- * The pacer ensures each slot (pipe:3..3+N-1) is written to at a steady
- * real-time rate (e.g. 10 ms = 480 samples @ 48kHz), even if WebRTC frames
- * arrive jittery, bursty, or with slightly different clocks.
- *
- * Key behavior:
- * - Writes exactly one frame per period, on a shared wall-clock grid.
- * - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
- *   never stalls.
- * - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
- * - Honors Node stream backpressure (`write()` return false) without breaking
- *   the timing grid.
- *
- * This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
- * can mix them without slow-downs or drift.
- *
- * Call this once right after spawning FFmpeg:
- * ```ts
- * const ff = spawnFFmpegProcess();
- * startPacer(ff, PARTICIPANT_SLOTS);
- * ```
- *
- * When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
- *
- * @param ff        Child process handle from spawn("ffmpeg", ...)
- * @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
- */
-function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
-    if (stopPacerFn) {
-        stopPacerFn();
-        stopPacerFn = null;
-    }
-    const writers = Array.from({ length: slotCount }, (_, i) => ff.stdio[3 + i]);
-    const nowMs = () => Number(process.hrtime.bigint()) / 1e6;
-    const outputFrameMs = (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms
-    const t0 = nowMs();
-    slots = Array.from({ length: slotCount }, () => ({
-        q: [],
-        lastFrames: FRAME_10MS_SAMPLES, // keep constant
-        nextDueMs: t0 + (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000,
-    }));
-    outputPacerState = {
-        frameQueue: [],
-        nextDueMs: t0 + outputFrameMs,
-        rtcAudioSource,
-        onAudioStreamReady,
-        didEmitReadyEvent: false,
-    };
-    const iv = setInterval(() => {
-        const t = nowMs();
-        for (let s = 0; s < slotCount; s++) {
-            const st = slots[s];
-            const w = writers[s];
-            const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms if 480, 20ms if 960
-            if (t >= st.nextDueMs) {
-                const buf = st.q.length ? st.q.shift() : Buffer.alloc(st.lastFrames * BYTES_PER_SAMPLE);
-                if (!w.write(buf)) {
-                    // Just continue without adding drain listener - backpressure will naturally resolve
+    /**
+     * Start the audio pacer loop for all input slots in an FFmpeg process.
+     *
+     * The pacer ensures each slot (pipe:3..3+N-1) is written to at a steady
+     * real-time rate (e.g. 10 ms = 480 samples @ 48kHz), even if WebRTC frames
+     * arrive jittery, bursty, or with slightly different clocks.
+     *
+     * Key behavior:
+     * - Writes exactly one frame per period, on a shared wall-clock grid.
+     * - Uses silence (zero-filled frame) if a slot's queue is empty, so timing
+     *   never stalls.
+     * - Resnaps the schedule if a slot switches between 10 ms / 20 ms frames.
+     * - Honors Node stream backpressure (`write()` return false) without breaking
+     *   the timing grid.
+     *
+     * This keeps all FFmpeg inputs phase-aligned and stable, so aresample/amix
+     * can mix them without slow-downs or drift.
+     *
+     * Call this once right after spawning FFmpeg:
+     * ```ts
+     * const ff = spawnFFmpegProcess();
+     * startPacer(ff, PARTICIPANT_SLOTS);
+     * ```
+     *
+     * When tearing down the mixer, always call `stopPacer()` before killing FFmpeg.
+     *
+     * @param ff        Child process handle from spawn("ffmpeg", ...)
+     * @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
+     */
+    function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
+        if (stopPacerFn) {
+            stopPacerFn();
+            stopPacerFn = null;
+        }
+        const writers = Array.from({ length: slotCount }, (_, i) => ff.stdio[3 + i]);
+        const nowMs = () => Number(process.hrtime.bigint()) / 1e6;
+        const outputFrameMs = (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms
+        const t0 = nowMs();
+        slots = Array.from({ length: slotCount }, () => ({
+            q: [],
+            lastFrames: FRAME_10MS_SAMPLES, // keep constant
+            nextDueMs: t0 + (FRAME_10MS_SAMPLES / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000,
+        }));
+        outputPacerState = {
+            frameQueue: [],
+            nextDueMs: t0 + outputFrameMs,
+            rtcAudioSource,
+            onAudioStreamReady,
+            didEmitReadyEvent: false,
+        };
+        const iv = setInterval(() => {
+            const t = nowMs();
+            for (let s = 0; s < slotCount; s++) {
+                const st = slots[s];
+                const w = writers[s];
+                const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000; // 10ms if 480, 20ms if 960
+                if (t >= st.nextDueMs) {
+                    const buf = st.q.length ? st.q.shift() : Buffer.alloc(st.lastFrames * BYTES_PER_SAMPLE);
+                    if (!w.write(buf)) {
+                        // Just continue without adding drain listener - backpressure will naturally resolve
+                        const late = t - st.nextDueMs;
+                        const steps = Math.max(1, Math.ceil(late / frameMs));
+                        st.nextDueMs += steps * frameMs;
+                        continue;
+                    }
                     const late = t - st.nextDueMs;
                     const steps = Math.max(1, Math.ceil(late / frameMs));
                     st.nextDueMs += steps * frameMs;
-                    continue;
                 }
-                const late = t - st.nextDueMs;
-                const steps = Math.max(1, Math.ceil(late / frameMs));
-                st.nextDueMs += steps * frameMs;
             }
-        }
-        if (!outputPacerState)
-            return;
-        // Handle output pacer for RTCAudioSource
-        const state = outputPacerState;
-        if (t >= state.nextDueMs) {
-            const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
-            if (!state.didEmitReadyEvent) {
-                state.onAudioStreamReady();
-                state.didEmitReadyEvent = true;
+            if (!outputPacerState)
+                return;
+            // Handle output pacer for RTCAudioSource
+            const state = outputPacerState;
+            if (t >= state.nextDueMs) {
+                const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
+                if (!state.didEmitReadyEvent) {
+                    state.onAudioStreamReady();
+                    state.didEmitReadyEvent = true;
+                }
+                state.rtcAudioSource.onData({
+                    samples: samples,
+                    sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
+                });
+                const late = t - state.nextDueMs;
+                const steps = Math.max(1, Math.ceil(late / outputFrameMs));
+                state.nextDueMs += steps * outputFrameMs;
             }
-            state.rtcAudioSource.onData({
-                samples: samples,
-                sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
-            });
-            const late = t - state.nextDueMs;
-            const steps = Math.max(1, Math.ceil(late / outputFrameMs));
-            state.nextDueMs += steps * outputFrameMs;
+        }, 5);
+        stopPacerFn = () => clearInterval(iv);
+    }
+    /**
+     * Stop the audio pacer loop and clear all input slots.
+     * Call this before killing the FFmpeg process to ensure clean shutdown.
+     */
+    function stopPacer() {
+        if (stopPacerFn)
+            stopPacerFn();
+        stopPacerFn = null;
+        slots = [];
+        slotBuffers.clear();
+        outputPacerState = null;
+    }
+    /**
+     * Queue a live frame for a given slot (0..N-1).
+     * Auto-resnaps the slot's schedule if the frame size (480/960) changes.
+     */
+    function enqueueFrame(slot, samples, numberOfFrames) {
+        const st = slots[slot];
+        if (!st)
+            return;
+        const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
+        st.q.push(buf);
+    }
+    /**
+     * Clear the audio queue for a specific slot when a participant leaves.
+     * This prevents stale audio data from continuing to play after disconnect.
+     */
+    function clearSlotQueue(slot) {
+        const st = slots[slot];
+        if (st) {
+            st.q = [];
+            slotBuffers.delete(slot);
+            const now = Number(process.hrtime.bigint()) / 1e6;
+            const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
+            st.nextDueMs = now + frameMs;
         }
-    }, 5);
-    stopPacerFn = () => clearInterval(iv);
-}
-/**
- * Stop the audio pacer loop and clear all input slots.
- * Call this before killing the FFmpeg process to ensure clean shutdown.
- */
-function stopPacer() {
-    if (stopPacerFn)
-        stopPacerFn();
-    stopPacerFn = null;
-    slots = [];
-}
-/**
- * Queue a live frame for a given slot (0..N-1).
- * Auto-resnaps the slot's schedule if the frame size (480/960) changes.
- */
-function enqueueFrame(slot, samples, numberOfFrames) {
-    const st = slots[slot];
-    if (!st)
-        return;
-    const buf = Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
-    st.q.push(buf);
-}
-/**
- * Clear the audio queue for a specific slot when a participant leaves.
- * This prevents stale audio data from continuing to play after disconnect.
- */
-function clearSlotQueue(slot) {
-    const st = slots[slot];
-    if (st) {
-        st.q = [];
-        const now = Number(process.hrtime.bigint()) / 1e6;
-        const frameMs = (st.lastFrames / STREAM_INPUT_SAMPLE_RATE_IN_HZ) * 1000;
-        st.nextDueMs = now + frameMs;
     }
-}
-/**
- * Get the FFmpeg arguments for mixing audio from multiple participants.
- * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
- * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
- */
-function getFFmpegArguments() {
-    const N = PARTICIPANT_SLOTS;
-    const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
-    const ffArgs = [];
-    for (let i = 0; i < N; i++) {
-        ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
-    }
-    const pre = [];
-    for (let i = 0; i < N; i++) {
-        pre.push(`[${i}:a]aresample=async=1:first_pts=0,asetpts=N/SR/TB[a${i}]`);
-    }
-    const labels = Array.from({ length: N }, (_, i) => `[a${i}]`).join("");
-    const amix = `${labels}amix=inputs=${N}:duration=longest:dropout_transition=250:normalize=0[mix]`;
-    const filter = `${pre.join(";")};${amix}`;
-    ffArgs.push("-hide_banner", "-nostats", "-loglevel", "error", "-filter_complex", filter, "-map", "[mix]", "-f", "s16le", "-ar", String(SR), "-ac", "1", "-c:a", "pcm_s16le", "pipe:1");
-    return ffArgs;
-}
-/**
- * Spawn a new FFmpeg process for mixing audio from multiple participants.
- * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
- * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
- * The process will log its output to stderr.
- * @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
- * @return The spawned FFmpeg process.
- */
-function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
-    const stdio = ["ignore", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
-    const args = getFFmpegArguments();
-    const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
-    startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
-    ffmpegProcess.stderr.setEncoding("utf8");
-    ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
-    ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
-    let audioBuffer = Buffer.alloc(0);
-    const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
-    ffmpegProcess.stdout.on("data", (chunk) => {
-        audioBuffer = Buffer.concat([audioBuffer, chunk]);
-        while (audioBuffer.length >= FRAME_SIZE_BYTES) {
-            const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
-            const samples = new Int16Array(FRAME_10MS_SAMPLES);
-            for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
-                samples[i] = frameData.readInt16LE(i * 2);
-            }
-            enqueueOutputFrame(samples);
-            audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
+    /**
+     * Get the FFmpeg arguments for debugging, which writes each participant's audio to a separate WAV file
+     * and also mixes them into a single WAV file.
+     * This is useful for inspecting the audio quality and timing of each participant.
+     */
+    function getFFmpegArgumentsDebug() {
+        const N = PARTICIPANT_SLOTS;
+        const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
+        const ffArgs = [];
+        for (let i = 0; i < N; i++) {
+            ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
         }
-    });
-    return ffmpegProcess;
-}
-/**
- * Write audio data from a MediaStreamTrack to the FFmpeg process.
- * This function creates an AudioSink for the track and sets up a data handler
- * that enqueues audio frames into the pacer.
- *
- * @param ffmpegProcess The FFmpeg process to which audio data will be written.
- * @param slot The participant slot number (0..N-1) to which this track belongs.
- * @param audioTrack The MediaStreamTrack containing the audio data.
- * @return An object containing the AudioSink, the writable stream, and a stop function.
- */
-function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
-    const writer = ffmpegProcess.stdio[3 + slot];
-    const sink = new AudioSink(audioTrack);
-    const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
-        if (ch !== 1 || bitsPerSample !== 16)
-            return;
-        let out = samples;
-        if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
-            const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
-            out = resampled;
-        }
-        appendAndDrainTo480(slot, out);
-    });
-    const stop = () => {
-        try {
-            unsubscribe();
-            sink.stop();
+        const pre = [];
+        for (let i = 0; i < N; i++) {
+            pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS,asplit=2[a${i}tap][a${i}mix]`);
         }
-        catch (_a) {
-            console.error("Failed to stop AudioSink");
+        const mixInputs = Array.from({ length: N }, (_, i) => `[a${i}mix]`).join("");
+        const filter = `${pre.join(";")};${mixInputs}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
+        ffArgs.push("-hide_banner", "-nostats", "-loglevel", "info", "-y", "-filter_complex", filter);
+        for (let i = 0; i < N; i++) {
+            ffArgs.push("-map", `[a${i}tap]`, "-f", "wav", "-c:a", "pcm_s16le", `pre${i}.wav`);
         }
-    };
-    return { sink, writer, stop };
-}
-/**
- * Stop the FFmpeg process and clean up all resources.
- * This function will unpipe the stdout, end all writable streams for each participant slot,
- * and kill the FFmpeg process.
- * @param ffmpegProcess The FFmpeg process to stop.
- */
-function stopFFmpegProcess(ffmpegProcess) {
-    stopPacer();
-    if (ffmpegProcess && !ffmpegProcess.killed) {
-        try {
-            ffmpegProcess.stdout.unpipe();
+        ffArgs.push("-map", "[mix]", "-f", "wav", "-c:a", "pcm_s16le", "mixed.wav");
+        return ffArgs;
+    }
+    /**
+     * Get the FFmpeg arguments for mixing audio from multiple participants.
+     * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
+     * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
+     */
+    function getFFmpegArguments() {
+        const N = PARTICIPANT_SLOTS;
+        const SR = STREAM_INPUT_SAMPLE_RATE_IN_HZ;
+        const ffArgs = [];
+        for (let i = 0; i < N; i++) {
+            ffArgs.push("-f", "s16le", "-ar", String(SR), "-ac", "1", "-i", `pipe:${3 + i}`);
         }
-        catch (_a) {
-            console.error("Failed to unpipe ffmpeg stdout");
+        const pre = [];
+        for (let i = 0; i < N; i++) {
+            pre.push(`[${i}:a]aresample=async=0:first_pts=0,asetpts=PTS-STARTPTS[a${i}]`);
         }
-        for (let i = 0; i < PARTICIPANT_SLOTS; i++) {
-            const w = ffmpegProcess.stdio[3 + i];
+        const labels = Array.from({ length: N }, (_, i) => `[a${i}]`).join("");
+        const amix = `${labels}amix=inputs=${N}:duration=first:dropout_transition=0:normalize=0[mix]`;
+        const filter = `${pre.join(";")};${amix}`;
+        ffArgs.push("-hide_banner", "-nostats", "-loglevel", "error", "-filter_complex", filter, "-map", "[mix]", "-f", "s16le", "-ar", String(SR), "-ac", "1", "-c:a", "pcm_s16le", "pipe:1");
+        return ffArgs;
+    }
+    /*
+     * Spawn a new FFmpeg process for debugging purposes.
+     * This will write each participant's audio to a separate WAV file and also mix them into a single WAV file.
+     * The output files will be named pre0.wav, pre1.wav, ..., and mixed.wav.
+     * The process will log its output to stderr.
+     * @return The spawned FFmpeg process.
+     */
+    function spawnFFmpegProcessDebug(rtcAudioSource, onAudioStreamReady) {
+        const stdio = ["ignore", "ignore", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
+        const args = getFFmpegArgumentsDebug();
+        const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
+        startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
+        ffmpegProcess.stderr.setEncoding("utf8");
+        ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
+        ffmpegProcess.on("error", () => console.error("FFmpeg process error (debug): is ffmpeg installed?"));
+        return ffmpegProcess;
+    }
+    /**
+     * Spawn a new FFmpeg process for mixing audio from multiple participants.
+     * This will read from the input pipes (3..3+N-1) and output a single mixed audio stream.
+     * The output is in PCM 16-bit little-endian format at 48kHz sample rate.
+     * The process will log its output to stderr.
+     * @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
+     * @return The spawned FFmpeg process.
+     */
+    function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
+        const stdio = ["pipe", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
+        const args = getFFmpegArguments();
+        const ffmpegProcess = child_process.spawn("ffmpeg", args, { stdio });
+        startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
+        ffmpegProcess.stderr.setEncoding("utf8");
+        ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
+        ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
+        let audioBuffer = Buffer.alloc(0);
+        const FRAME_SIZE_BYTES = FRAME_10MS_SAMPLES * BYTES_PER_SAMPLE; // 480 samples * 2 bytes = 960 bytes
+        ffmpegProcess.stdout.on("data", (chunk) => {
+            audioBuffer = Buffer.concat([audioBuffer, chunk]);
+            while (audioBuffer.length >= FRAME_SIZE_BYTES) {
+                const frameData = audioBuffer.subarray(0, FRAME_SIZE_BYTES);
+                const samples = new Int16Array(FRAME_10MS_SAMPLES);
+                for (let i = 0; i < FRAME_10MS_SAMPLES; i++) {
+                    samples[i] = frameData.readInt16LE(i * 2);
+                }
+                enqueueOutputFrame(samples);
+                audioBuffer = audioBuffer.subarray(FRAME_SIZE_BYTES);
+            }
+        });
+        return ffmpegProcess;
+    }
+    /**
+     * Write audio data from a MediaStreamTrack to the FFmpeg process.
+     * This function creates an AudioSink for the track and sets up a data handler
+     * that enqueues audio frames into the pacer.
+     *
+     * @param ffmpegProcess The FFmpeg process to which audio data will be written.
+     * @param slot The participant slot number (0..N-1) to which this track belongs.
+     * @param audioTrack The MediaStreamTrack containing the audio data.
+     * @return An object containing the AudioSink, the writable stream, and a stop function.
+     */
+    function writeAudioDataToFFmpeg(ffmpegProcess, slot, audioTrack) {
+        const writer = ffmpegProcess.stdio[3 + slot];
+        const sink = new AudioSink(audioTrack);
+        const unsubscribe = sink.subscribe(({ samples, sampleRate: sr, channelCount: ch, bitsPerSample, numberOfFrames }) => {
+            if (ch !== 1 || bitsPerSample !== 16)
+                return;
+            let out = samples;
+            if (sr !== STREAM_INPUT_SAMPLE_RATE_IN_HZ) {
+                const resampled = resampleTo48kHz(samples, sr, numberOfFrames !== null && numberOfFrames !== void 0 ? numberOfFrames : samples.length);
+                out = resampled;
+            }
+            appendAndDrainTo480(slot, out);
+        });
+        const stop = () => {
+            try {
+                unsubscribe();
+                sink.stop();
+            }
+            catch (_a) {
+                console.error("Failed to stop AudioSink");
+            }
+        };
+        return { sink, writer, stop };
+    }
+    /**
+     * Stop the FFmpeg process and clean up all resources.
+     * This function will unpipe the stdout, end all writable streams for each participant slot,
+     * and kill the FFmpeg process.
+     * @param ffmpegProcess The FFmpeg process to stop.
+     */
+    function stopFFmpegProcess(ffmpegProcess) {
+        var _a, _b;
+        stopPacer();
+        if (ffmpegProcess && !ffmpegProcess.killed) {
+            try {
+                ffmpegProcess.stdout.unpipe();
+            }
+            catch (_c) {
+                console.error("Failed to unpipe ffmpeg stdout");
+            }
+            for (let i = 0; i < PARTICIPANT_SLOTS; i++) {
+                const w = ffmpegProcess.stdio[3 + i];
+                try {
+                    w.end();
+                }
+                catch (_d) {
+                    console.error("Failed to end ffmpeg writable stream");
+                }
+            }
             try {
-                w.end();
+                (_a = ffmpegProcess.stdin) === null || _a === void 0 ? void 0 : _a.write("q\n");
+                (_b = ffmpegProcess.stdin) === null || _b === void 0 ? void 0 : _b.end();
             }
-            catch (_b) {
-                console.error("Failed to end ffmpeg writable stream");
+            catch (_e) {
+                console.error("Failed to end ffmpeg stdin");
             }
         }
-        ffmpegProcess.kill("SIGTERM");
     }
+    return {
+        spawnFFmpegProcess,
+        spawnFFmpegProcessDebug,
+        writeAudioDataToFFmpeg,
+        stopFFmpegProcess,
+        clearSlotQueue,
+    };
 }
 
 class AudioMixer extends EventEmitter.EventEmitter {
@@ -404,6 +469,7 @@ class AudioMixer extends EventEmitter.EventEmitter {
         this.rtcAudioSource = null;
         this.participantSlots = new Map();
         this.activeSlots = {};
+        this.mixer = createFfmpegMixer();
         this.setupMediaStream();
         this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
         this.onStreamReady = onStreamReady;
@@ -422,7 +488,7 @@ class AudioMixer extends EventEmitter.EventEmitter {
             return;
         }
         if (!this.ffmpegProcess && this.rtcAudioSource) {
-            this.ffmpegProcess = spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
+            this.ffmpegProcess = this.mixer.spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
         }
         for (const p of participants)
             this.attachParticipantIfNeeded(p);
@@ -435,7 +501,7 @@ class AudioMixer extends EventEmitter.EventEmitter {
     }
     stopAudioMixer() {
         if (this.ffmpegProcess) {
-            stopFFmpegProcess(this.ffmpegProcess);
+            this.mixer.stopFFmpegProcess(this.ffmpegProcess);
             this.ffmpegProcess = null;
         }
         this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
@@ -488,7 +554,7 @@ class AudioMixer extends EventEmitter.EventEmitter {
             }
             this.activeSlots[slot] = undefined;
         }
-        const { sink, writer, stop } = writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
+        const { sink, writer, stop } = this.mixer.writeAudioDataToFFmpeg(this.ffmpegProcess, slot, audioTrack);
         this.activeSlots[slot] = { sink, writer, stop, trackId: audioTrack.id };
         (_a = audioTrack.addEventListener) === null || _a === void 0 ? void 0 : _a.call(audioTrack, "ended", () => this.detachParticipant(participantId));
     }
@@ -507,7 +573,7 @@ class AudioMixer extends EventEmitter.EventEmitter {
             this.activeSlots[slot] = undefined;
         }
         // Clear any queued audio data for this slot to prevent stale audio
-        clearSlotQueue(slot);
+        this.mixer.clearSlotQueue(slot);
         this.participantSlots.set(slot, "");
     }
 }
@@ -518,6 +584,7 @@ class Assistant extends EventEmitter {
         this.mediaStream = null;
         this.audioSource = null;
         this.combinedStream = null;
+        this.remoteMediaTracks = {};
         this.roomUrl = null;
         this.handleConnectionStatusChange = (status) => {
             if (status === "connected") {
@@ -527,6 +594,41 @@ class Assistant extends EventEmitter {
                 this.emit(ASSISTANT_LEFT_ROOM, { roomUrl: this.roomUrl || "" });
             }
         };
+        this.handleRemoteParticipantsTracksChange = (remoteParticipants) => {
+            const currentRemoteMediaTracks = remoteParticipants.flatMap(({ id: participantId, stream }) => {
+                if (!stream) {
+                    return [];
+                }
+                const tracks = stream.getTracks();
+                tracks.forEach((track) => {
+                    if (!this.remoteMediaTracks[track.id]) {
+                        const eventName = track.kind === "video" ? PARTICIPANT_VIDEO_TRACK_ADDED : PARTICIPANT_AUDIO_TRACK_ADDED;
+                        this.emit(eventName, {
+                            participantId,
+                            stream,
+                            track,
+                        });
+                        this.remoteMediaTracks[track.id] = {
+                            participantId,
+                            stream,
+                            track,
+                        };
+                    }
+                });
+                return tracks;
+            });
+            Object.values(this.remoteMediaTracks).forEach(({ participantId, stream, track }) => {
+                if (!currentRemoteMediaTracks.includes(track)) {
+                    const eventName = track.kind === "video" ? PARTICIPANT_VIDEO_TRACK_REMOVED : PARTICIPANT_AUDIO_TRACK_REMOVED;
+                    this.emit(eventName, {
+                        participantId,
+                        stream,
+                        track,
+                    });
+                    delete this.remoteMediaTracks[track.id];
+                }
+            });
+        };
         this.assistantKey = assistantKey;
         this.client = new core.WherebyClient();
         this.roomConnection = this.client.getRoomConnection();
@@ -551,8 +653,9 @@ class Assistant extends EventEmitter {
             const audioMixer = new AudioMixer(handleStreamReady);
             this.combinedStream = audioMixer.getCombinedAudioStream();
             this.roomConnection.subscribeToRemoteParticipants(audioMixer.handleRemoteParticipants.bind(audioMixer));
-            this.roomConnection.subscribeToConnectionStatus(this.handleConnectionStatusChange);
         }
+        this.roomConnection.subscribeToConnectionStatus(this.handleConnectionStatusChange);
+        this.roomConnection.subscribeToRemoteParticipants(this.handleRemoteParticipantsTracksChange);
     }
     joinRoom(roomUrl) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -669,7 +772,7 @@ const webhookRouter = (webhookTriggers, emitter) => {
         res.status(200);
         res.end();
     });
-    router.post("/", jsonParser, (req, res) => {
+    router.post("/", jsonParser, (req, res) => __awaiter(void 0, void 0, void 0, function* () {
         var _a, _b, _c, _d, _e, _f, _g, _h;
         assert(req.body, "message body is required");
         assert("type" in req.body, "webhook type is required");
@@ -677,19 +780,19 @@ const webhookRouter = (webhookTriggers, emitter) => {
         switch (req.body.type) {
             case "room.client.joined":
                 shouldTriggerOnReceivedWebhook =
-                    (_b = (_a = webhookTriggers["room.client.joined"]) === null || _a === void 0 ? void 0 : _a.call(webhookTriggers, req.body)) !== null && _b !== void 0 ? _b : false;
+                    (_b = (yield ((_a = webhookTriggers["room.client.joined"]) === null || _a === void 0 ? void 0 : _a.call(webhookTriggers, req.body)))) !== null && _b !== void 0 ? _b : false;
                 break;
             case "room.client.left":
                 shouldTriggerOnReceivedWebhook =
-                    (_d = (_c = webhookTriggers["room.client.left"]) === null || _c === void 0 ? void 0 : _c.call(webhookTriggers, req.body)) !== null && _d !== void 0 ? _d : false;
+                    (_d = (yield ((_c = webhookTriggers["room.client.left"]) === null || _c === void 0 ? void 0 : _c.call(webhookTriggers, req.body)))) !== null && _d !== void 0 ? _d : false;
                 break;
             case "room.session.started":
                 shouldTriggerOnReceivedWebhook =
-                    (_f = (_e = webhookTriggers["room.session.started"]) === null || _e === void 0 ? void 0 : _e.call(webhookTriggers, req.body)) !== null && _f !== void 0 ? _f : false;
+                    (_f = (yield ((_e = webhookTriggers["room.session.started"]) === null || _e === void 0 ? void 0 : _e.call(webhookTriggers, req.body)))) !== null && _f !== void 0 ? _f : false;
                 break;
             case "room.session.ended":
                 shouldTriggerOnReceivedWebhook =
-                    (_h = (_g = webhookTriggers["room.session.ended"]) === null || _g === void 0 ? void 0 : _g.call(webhookTriggers, req.body)) !== null && _h !== void 0 ? _h : false;
+                    (_h = (yield ((_g = webhookTriggers["room.session.ended"]) === null || _g === void 0 ? void 0 : _g.call(webhookTriggers, req.body)))) !== null && _h !== void 0 ? _h : false;
                 break;
         }
         if (shouldTriggerOnReceivedWebhook) {
@@ -698,7 +801,7 @@ const webhookRouter = (webhookTriggers, emitter) => {
         }
         res.status(200);
         res.end();
-    });
+    }));
     return router;
 };
 class Trigger extends EventEmitter.EventEmitter {
@@ -711,12 +814,9 @@ class Trigger extends EventEmitter.EventEmitter {
         const app = express();
         const router = webhookRouter(this.webhookTriggers, this);
         app.use(router);
-        const server = app.listen(this.port, () => {
+        app.listen(this.port, () => {
             // console.log(`Bot trigger server now running on port[${this.port}]`);
         });
-        process.on("SIGTERM", () => {
-            server.close();
-        });
     }
 }
 
@@ -726,5 +826,9 @@ exports.AUDIO_STREAM_READY = AUDIO_STREAM_READY;
 exports.Assistant = Assistant;
 exports.AudioSink = AudioSink;
 exports.AudioSource = AudioSource;
+exports.PARTICIPANT_AUDIO_TRACK_ADDED = PARTICIPANT_AUDIO_TRACK_ADDED;
+exports.PARTICIPANT_AUDIO_TRACK_REMOVED = PARTICIPANT_AUDIO_TRACK_REMOVED;
+exports.PARTICIPANT_VIDEO_TRACK_ADDED = PARTICIPANT_VIDEO_TRACK_ADDED;
+exports.PARTICIPANT_VIDEO_TRACK_REMOVED = PARTICIPANT_VIDEO_TRACK_REMOVED;
 exports.TRIGGER_EVENT_SUCCESS = TRIGGER_EVENT_SUCCESS;
 exports.Trigger = Trigger;