@pexip/media-processor 16.7.1 → 17.1.0

package/dist/main/audio.js

@@ -0,0 +1,653 @@
+import { copyByteBufferToFloatBuffer, processAverageVolume } from './process';
+import { createDenoiseWorkletNode } from './workletNodes';
+import { createAnalyserNode, createChannelSplitterNode, createGainNode, createMediaStreamAudioDestinationNode, createMediaStreamAudioSourceNode, createMediaStreamAudioClone, createDelayNode, muteToGain, stopStreamTracks, createMediaElementSourceNode, createChannelMergerNode, } from './utils';
+import { isAudioParam, isAudioNodeInit } from './typeGuards';
+/**
+ * A function to create `AudioContext` using constructor or factory function
+ * depends on the browser supports
+ *
+ * @param options - @see {@link AudioContextOptions}
+ *
+ * @internal
+ */
+export const createAudioContext = (options) => {
+    try {
+        return new AudioContext(options);
+    }
+    catch {
+        return new window.webkitAudioContext(options);
+    }
+};
+/**
+ * Resume the stream whenever interrupted
+ *
+ * @param audioContext - AudioContext
+ *
+ * @alpha
+ */
+export function resumeAudioOnInterruption(audioContext) {
+    const resumeInterrupted = () => {
+        // Resume the stream whenever Safari has interrupted it
+        if (audioContext.state === 'interrupted') {
+            void audioContext.resume();
+        }
+    };
+    audioContext.addEventListener('statechange', resumeInterrupted);
+    return () => {
+        audioContext.removeEventListener('statechange', resumeInterrupted);
+    };
+}
+/**
+ * Resume the AudioContext whenever the source track is unmuted
+ *
+ * @param audioContext - The `AudioContext` to resume
+ *
+ * @alpha
+ */
+export const resumeAudioOnUnmute = (context) => 
+/**
+ * @param track - The source track to listen on the `unmute` event
+ */
+(track) => {
+    const resume = () => {
+        if (context.state === 'suspended') {
+            void context.resume();
+        }
+    };
+    track.addEventListener('unmute', resume);
+    return () => {
+        track.removeEventListener('unmute', resume);
+    };
+};
+/**
+ * Wrap GainNode with mute function
+ *
+ * @param context - @see {@link AudioContext}
+ * @param muteInit - initial mute state of the node
+ *
+ * @internal
+ */
+const createGainWithMute = (context, muteInit = false) => {
+    let mute = muteInit;
+    const gain = createGainNode(context, {
+        gain: muteToGain(mute),
+        channelCountMode: 'explicit',
+    });
+    const disconnect = gain.disconnect.bind(gain);
+    const connect = gain.connect.bind(gain);
+    return {
+        get node() {
+            return gain;
+        },
+        get mute() {
+            return mute;
+        },
+        set mute(shouldMute) {
+            mute = shouldMute;
+            const time = context.currentTime;
+            if (isFinite(time)) {
+                gain.gain.setValueAtTime(muteToGain(shouldMute), time);
+            }
+            else {
+                throw new Error('Set gain with non-finite time value');
+            }
+        },
+        connect,
+        disconnect,
+    };
+};
+/**
+ * Wrap AnalyserNode with polyfill and able to get average volume
+ *
+ * @param context - @see {@link AudioContext}
+ * @param options - @see {@link AnalyserOptions}
+ *
+ * @internal
+ */
+const createAnalyzer = (context, options) => {
+    const analyser = createAnalyserNode(context, options);
+    const getByteFrequencyData = analyser.getByteFrequencyData.bind(analyser);
+    const getByteTimeDomainData = analyser.getByteTimeDomainData.bind(analyser);
+    const getFloatFrequencyData = analyser.getFloatFrequencyData.bind(analyser);
+    const connect = analyser.connect.bind(analyser);
+    const disconnect = analyser.disconnect.bind(analyser);
+    let byteBuffer;
+    /**
+     * Safari does not support getFloatTimeDomainData API, thus polyfill is
+     * needed
+     *
+     * According to https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/getFloatTimeDomainData
+     */
+    const getFloatTimeDomainData = (buffer) => {
+        if ('getFloatTimeDomainData' in AnalyserNode.prototype) {
+            return analyser.getFloatTimeDomainData(buffer);
+        }
+        // Use getByteTimeDomainData and convert the result to the buffer
+        if (!byteBuffer) {
+            byteBuffer = new Uint8Array(buffer.length);
+        }
+        analyser.getByteTimeDomainData(byteBuffer);
+        return copyByteBufferToFloatBuffer(byteBuffer, buffer);
+    };
+    const getAverageVolume = (buffer) => {
+        getFloatTimeDomainData(buffer);
+        return processAverageVolume(Array.from(buffer));
+    };
+    return {
+        get node() {
+            return analyser;
+        },
+        get frequencyBinCount() {
+            return analyser.frequencyBinCount;
+        },
+        get fftSize() {
+            return analyser.fftSize;
+        },
+        set fftSize(size) {
+            analyser.fftSize = size;
+        },
+        get minDecibels() {
+            return analyser.minDecibels;
+        },
+        set minDecibels(decibels) {
+            analyser.minDecibels = decibels;
+        },
+        get maxDecibels() {
+            return analyser.maxDecibels;
+        },
+        set maxDecibels(decibels) {
+            analyser.maxDecibels = decibels;
+        },
+        get smoothingTimeConstant() {
+            return analyser.smoothingTimeConstant;
+        },
+        set smoothingTimeConstant(constant) {
+            analyser.smoothingTimeConstant = constant;
+        },
+        getByteTimeDomainData,
+        getByteFrequencyData,
+        getFloatFrequencyData,
+        getFloatTimeDomainData,
+        getAverageVolume,
+        connect,
+        disconnect,
+    };
+};
+/**
+ * Subscribe MessagePort message from an AudioWorkletNode
+ *
+ * @param workletNode - the node to subscribe
+ * @param options - can pass a message handler here to handle the message
+ */
+export const subscribeWorkletNode = (workletNode, { messageHandler, errorHandler } = {}) => {
+    const subscribeMessage = (event) => {
+        messageHandler?.(event.data);
+    };
+    const subscribeToPortError = (event) => {
+        if (errorHandler) {
+            errorHandler(event);
+        }
+    };
+    const subscribeToProcessorError = (event) => {
+        if (errorHandler) {
+            errorHandler(event);
+        }
+    };
+    workletNode.port.addEventListener('message', subscribeMessage);
+    workletNode.port.start();
+    workletNode.addEventListener('processorerror', subscribeToProcessorError);
+    workletNode.port.addEventListener('messageerror', subscribeToPortError);
+    return () => {
+        workletNode.port.postMessage({ type: 'release' });
+        workletNode.port.removeEventListener('message', subscribeMessage);
+        workletNode.removeEventListener('processorerror', subscribeToProcessorError);
+        workletNode.port.removeEventListener('messageerror', subscribeToPortError);
+    };
+};
+/**
+ * Offset factor to compensate real-time audio processing since `setTimeout` is
+ * not guaranteed to execute at provided delay and we need this adjustment
+ * factor to compensate the delay to get a result similar what we did with
+ * `AudioWorklet`.
+ */
+const TIMEOUT_DELAY_ADJUSTMENT = 0.5;
+/**
+ * Subscribe to a timeout loop to get the data from Analyzer
+ *
+ * @param analyzer - the analyzer to subscribe
+ * @param options - message handler, etc.
+ */
+export const subscribeTimeoutAnalyzerNode = (analyzer, { messageHandler, updateFrequency = 2.0 }) => {
+    // A timeout-based approach
+    const timeoutMs = updateFrequency * 1000 * TIMEOUT_DELAY_ADJUSTMENT;
+    let timeoutId = 0;
+    let stopTimeout = false;
+    const clearTimeout = () => {
+        if (timeoutId) {
+            window.clearTimeout(timeoutId);
+            timeoutId = 0;
+        }
+    };
+    const process = () => {
+        messageHandler(analyzer);
+        clearTimeout();
+        if (!stopTimeout) {
+            timeoutId = window.setTimeout(process, timeoutMs);
+        }
+    };
+    process();
+    return () => {
+        stopTimeout = true;
+        clearTimeout();
+    };
+};
+function createBaseAudioNode(name, create, release) {
+    const props = {
+        name,
+        node: undefined,
+        audioNode: undefined,
+        outputs: new WeakSet(),
+    };
+    const createNode = (context, prevNode) => {
+        const [audioNode, node] = create(context, prevNode);
+        props.audioNode = audioNode;
+        props.node = node;
+        return [audioNode, node];
+    };
+    const releaseNode = () => {
+        release?.();
+        props.outputs = new WeakSet();
+        props.audioNode?.disconnect();
+        props.audioNode = undefined;
+    };
+    const connectNode = (param) => {
+        if (!param) {
+            return;
+        }
+        if (!props.audioNode) {
+            throw new Error('Source AudioNode is not initialized');
+        }
+        const [destination, output, input] = Array.isArray(param)
+            ? param
+            : [param];
+        if (isAudioParam(destination)) {
+            props.outputs.add(destination);
+            return props.audioNode.connect(destination, output);
+        }
+        if (isAudioNodeInit(destination)) {
+            if (!destination.audioNode) {
+                throw new Error('Destination AudioNode is not initialized');
+            }
+            props.outputs.add(destination);
+            return props.audioNode.connect(destination.audioNode, output, input);
+        }
+    };
+    const disconnectNode = (destInit) => {
+        if (!destInit) {
+            return;
+        }
+        if (!props.audioNode) {
+            throw new Error('AudioNode is not initialized');
+        }
+        const [destination, output, input] = Array.isArray(destInit)
+            ? destInit
+            : [destInit];
+        if (destination && !props.outputs.has(destination)) {
+            return; // There is no connection between the nodes
+        }
+        if (isAudioParam(destination)) {
+            props.outputs.delete(destination);
+            if (output !== undefined) {
+                return props.audioNode.disconnect(destination, output);
+            }
+            return props.audioNode.disconnect(destination);
+        }
+        if (isAudioNodeInit(destination)) {
+            if (!destination.audioNode) {
+                throw new Error('Destination AudioNode is not initialized');
+            }
+            props.outputs.delete(destination);
+            if (output !== undefined) {
+                if (input !== undefined) {
+                    return props.audioNode.disconnect(destination.audioNode, output, input);
+                }
+                return props.audioNode.disconnect(destination.audioNode, output);
+            }
+            return props.audioNode.disconnect(destination.audioNode);
+        }
+        props.outputs = new WeakSet();
+        props.audioNode.disconnect();
+    };
+    const hasConnectedTo = init => {
+        return props.outputs.has(init);
+    };
+    return Object.assign(props, {
+        create: createNode,
+        connect: connectNode,
+        disconnect: disconnectNode,
+        release: releaseNode,
+        hasConnectedTo,
+        toJSON: () => ({
+            name: props.name,
+            node: props.node,
+            audioNode: props.audioNode,
+        }),
+    });
+}
+/**
+ * Create a MediaStreamAudioSourceNodeInit
+ *
+ * @param mediaStream - Source MediaStream
+ * @param shouldResetEnabled - Whether or not to enable the cloned track
+ */
+export const createStreamSourceGraphNode = (mediaStream, shouldResetEnabled = true) => {
+    let stream = undefined;
+    let unsubscribeUnmutes = undefined;
+    return createBaseAudioNode('source', context => {
+        stream = createMediaStreamAudioClone(mediaStream);
+        // Enabled the track for the cloned stream
+        if (shouldResetEnabled) {
+            stream
+                .getAudioTracks()
+                .forEach(track => (track.enabled = true));
+        }
+        unsubscribeUnmutes = mediaStream
+            .getAudioTracks()
+            .map(resumeAudioOnUnmute(context));
+        const node = createMediaStreamAudioSourceNode(context, {
+            mediaStream: stream,
+        });
+        return [node, node];
+    }, () => {
+        stopStreamTracks(stream);
+        unsubscribeUnmutes?.forEach(unsubscribe => unsubscribe());
+        stream = undefined;
+    });
+};
+/**
+ * Create a MediaStreamAudioSourceNodeInit
+ *
+ * @param mediaStream - Source MediaStream
+ */
+export const createMediaElementSourceGraphNode = (mediaElement) => {
+    return createBaseAudioNode('source', context => {
+        const node = createMediaElementSourceNode(context, {
+            mediaElement,
+        });
+        return [node, node];
+    });
+};
+/**
+ * Create an analyzer node with push-based subscription
+ */
+export const createAnalyzerSubscribableGraphNode = ({ messageHandler, updateFrequency = 2.0, ...analyserOptions }) => {
+    let unsubscribe;
+    return createBaseAudioNode('analyzer', context => {
+        const node = createAnalyzer(context, analyserOptions);
+        // Subscribe to the timeout callback
+        unsubscribe = subscribeTimeoutAnalyzerNode(node, {
+            messageHandler,
+            updateFrequency,
+        });
+        return [node.node, node];
+    }, () => {
+        unsubscribe?.();
+        unsubscribe = undefined;
+    });
+};
+/**
+ * Create a noise suppression node
+ *
+ * @param data - WebAssembly source
+ */
+export const createDenoiseWorkletGraphNode = (data, messageHandler) => {
+    let unsubscribe;
+    return createBaseAudioNode('denoise', (context, prevNode) => {
+        const sampleRate = context.sampleRate;
+        const channelCount = prevNode?.channelCount ?? 1;
+        const node = createDenoiseWorkletNode(context, {
+            outputChannelCount: [channelCount],
+            processorOptions: {
+                data,
+                sampleRate,
+                shouldSendVAD: !!messageHandler,
+            },
+        });
+        unsubscribe = subscribeWorkletNode(node, { messageHandler });
+        return [node, node];
+    }, () => {
+        unsubscribe?.();
+        unsubscribe = undefined;
+    });
+};
+/**
+ * Create a GainNodeInit
+ *
+ * @param mute - initial mute state
+ */
+export const createGainGraphNode = (mute) => {
+    return createBaseAudioNode('gain', context => {
+        const node = createGainWithMute(context, mute);
+        return [node.node, node];
+    });
+};
+/**
+ * Create an AnalyzerNodeInit
+ *
+ * @param options - @see {@link AnalyserOptions}
+ */
+export const createAnalyzerGraphNode = (options) => {
+    return createBaseAudioNode('analyzer', context => {
+        const node = createAnalyzer(context, options);
+        return [node.node, node];
+    });
+};
+/**
+ * Create a MediaStreamAudioDestinationNodeInit
+ */
+export const createStreamDestinationGraphNode = (options) => {
+    let node = undefined;
+    return createBaseAudioNode('destination', context => {
+        node = createMediaStreamAudioDestinationNode(context, options);
+        return [node, node];
+    }, () => {
+        stopStreamTracks(node?.stream);
+        node = undefined;
+    });
+};
+/**
+ * Create an `AudioDestinationNode`
+ */
+export const createAudioDestinationGraphNode = () => {
+    return createBaseAudioNode('destination', context => {
+        const node = context.destination;
+        return [node, node];
+    });
+};
+/**
+ * Create a `DelayNode`
+ *
+ * @param options - @see DelayOptions
+ */
+export const createDelayGraphNode = (options) => {
+    return createBaseAudioNode('delay', context => {
+        const node = createDelayNode(context, options);
+        return [node, node];
+    });
+};
+/**
+ * Create a ChannelSplitterNode
+ *
+ * @param options - @see ChannelSplitterOptions
+ */
+export const createChannelSplitterGraphNode = (options) => {
+    return createBaseAudioNode('splitter', context => {
+        const node = createChannelSplitterNode(context, options);
+        return [node, node];
+    });
+};
+/**
+ * Create a ChannelMergerNode
+ *
+ * @param options - @see ChannelMergerOptions
+ */
+export const createChannelMergerGraphNode = (options) => {
+    return createBaseAudioNode('merger', context => {
+        const node = createChannelMergerNode(context, options);
+        return [node, node];
+    });
+};
+const createNodeInitDisconnector = () => (srcInit, destInit) => {
+    if (!srcInit) {
+        return;
+    }
+    const [source, outputIdx, inputIdx] = Array.isArray(srcInit)
+        ? srcInit
+        : [srcInit];
+    if (!source) {
+        return;
+    }
+    if (isAudioParam(source)) {
+        throw new Error('AudioParam cannot be used as the source of disconnect');
+    }
+    const [destination] = Array.isArray(destInit) ? destInit : [destInit];
+    return source.disconnect([destination, outputIdx, inputIdx]);
+};
+const createNodeInitConnector = (context) => (srcInit, destInit) => {
+    if (!srcInit || !destInit) {
+        return;
+    }
+    const [source, outputIdx, inputIdx] = Array.isArray(srcInit)
+        ? srcInit
+        : [srcInit];
+    const [destination] = Array.isArray(destInit) ? destInit : [destInit];
+    if (!source || !destination) {
+        return;
+    }
+    if (isAudioParam(source)) {
+        throw new Error('AudioParam cannot be used as the source of connect');
+    }
+    const [sourceNode] = source.audioNode
+        ? [source.audioNode]
+        : source.create(context);
+    if (isAudioNodeInit(destination) && !destination.audioNode) {
+        destination.create(context, sourceNode);
+    }
+    return source.connect([destination, outputIdx, inputIdx]);
+};
+/**
+ * Accepts AudioNodeInitConnections to build the audio graph within a signal audio context
+ *
+ * @param initialConnections - A list of AudioNodeInit to build the graph in a linear fashion
+ * @param options - @see {@link AudioGraphOptions}
+ *
+ * @example
+ *
+ * ```typescript
+ * const source = createStreamSourceGraphNode(stream);
+ * const analyzer = createAnalyzerGraphNode({fftSize});
+ * const audioGraph = createAudioGraph([[source, analyzer]]);
+ * ```
+ */
+export const createAudioGraph = (initialConnections, options = {}) => {
+    const props = {
+        closing: false,
+        inits: new Set(),
+    };
+    const audioContext = options.context ?? createAudioContext(options.contextOptions);
+    const unsubscribeStateChanged = resumeAudioOnInterruption(audioContext);
+    const connectInit = createNodeInitConnector(audioContext);
+    const disconnectInit = createNodeInitDisconnector();
+    const connect = sequence => {
+        sequence.forEach((initParam, idx, initParams) => {
+            const prevParam = initParams[idx - 1];
+            const [currentInit] = Array.isArray(initParam)
+                ? initParam
+                : [initParam];
+            connectInit(prevParam, currentInit);
+            if (isAudioNodeInit(currentInit)) {
+                // Keep a reference
+                props.inits.add(currentInit);
+            }
+        });
+    };
+    const disconnect = sequence => {
+        sequence.forEach((initParam, idx, initParams) => {
+            const prevParam = initParams[idx - 1];
+            const [currentInit] = Array.isArray(initParam)
+                ? initParam
+                : [initParam];
+            disconnectInit(prevParam, currentInit);
+        });
+    };
+    const addWorklet = async (moduleURL, options) => {
+        if (!props.workletModule) {
+            await audioContext.audioWorklet.addModule(moduleURL, options);
+            props.workletModule = { moduleURL, options };
+        }
+    };
+    // Initial connections setup
+    initialConnections.forEach(connect);
+    const releaseInit = init => {
+        if (props.inits.has(init)) {
+            init.release();
+            props.inits.delete(init);
+        }
+    };
+    return {
+        get inits() {
+            return Array.from(props.inits);
+        },
+        get context() {
+            return audioContext;
+        },
+        get state() {
+            return props.closing ? 'closing' : audioContext.state;
+        },
+        connect,
+        disconnect,
+        addWorklet,
+        releaseInit,
+        release: () => {
+            return new Promise(resolve => {
+                if (!props.closing && audioContext.state !== 'closed') {
+                    props.closing = true;
+                    props.workletModule = undefined;
+                    unsubscribeStateChanged();
+                    void audioContext.close().then(() => {
+                        props.closing = false;
+                        props.inits.forEach(releaseInit);
+                        resolve();
+                    });
+                }
+                else {
+                    resolve();
+                }
+            });
+        },
+    };
+};
+export const createAudioGraphProxy = (audioGraph, handlers) => new Proxy(audioGraph, {
+    get: (target, p) => {
+        switch (p) {
+            case 'connect': {
+                const r = target[p];
+                return (...args) => {
+                    handlers.connect?.(target, args);
+                    return r.apply(target, args);
+                };
+            }
+            case 'disconnect': {
+                const r = target[p];
+                return (...args) => {
+                    handlers.disconnect?.(target, args);
+                    return r.apply(target, args);
+                };
+            }
+            default: {
+                return target[p];
+            }
+        }
+    },
+    set: () => false,
+});

package/dist/main/benchUtils.d.ts

@@ -0,0 +1,12 @@
+import type { Clock } from './types';
+export interface Benchmark {
+    begin(): void;
+    end(): void;
+    calculateFps(): number;
+}
+interface BenchmarkOptions {
+    calculationThresholdMS?: number;
+}
+export declare const createBenchmark: (clock?: Clock, { calculationThresholdMS }?: BenchmarkOptions) => Benchmark;
+export declare const calculateFps: (time: number) => number;
+export {};

package/dist/main/benchUtils.js

@@ -0,0 +1,34 @@
+export const createBenchmark = (clock = performance, { calculationThresholdMS = 1000 } = {}) => {
+    const props = {
+        beginTime: 0,
+        endTime: 0,
+        sumDelta: 0,
+        count: 0,
+        lastCalculateTime: 0,
+        lastResult: 0,
+    };
+    return {
+        begin: () => {
+            props.beginTime = clock.now();
+        },
+        end: () => {
+            props.endTime = clock.now();
+            props.sumDelta += props.endTime - props.beginTime;
+            ++props.count;
+        },
+        calculateFps: () => {
+            const calculateTime = clock.now();
+            if (calculateTime - props.lastCalculateTime >=
+                calculationThresholdMS) {
+                const result = props.count === 0 ? 0 : props.sumDelta / props.count;
+                props.sumDelta = 0;
+                props.count = 0;
+                props.lastCalculateTime = calculateTime;
+                props.lastResult = result;
+                return result;
+            }
+            return props.lastResult;
+        },
+    };
+};
+export const calculateFps = (time) => time === 0 ? time : 1000.0 / time;

package/dist/main/generator.d.ts

@@ -0,0 +1,4 @@
+export declare const createMediaStreamTrackGenerator: {
+    (init: MediaStreamAudioTrackGeneratorInit): MediaStreamAudioTrackGenerator;
+    (init: MediaStreamVideoTrackGeneratorInit): MediaStreamVideoTrackGenerator;
+};

package/dist/main/generator.js

@@ -0,0 +1,4 @@
+export const createMediaStreamTrackGenerator = init => {
+    const generator = new MediaStreamTrackGenerator(init);
+    return generator;
+};