@pexip/media-processor 16.7.1 → 17.1.0

package/dist/main/process.js

@@ -0,0 +1,364 @@
+import { rms, round } from './math';
+import { throttleProcess } from './utils';
+// Constants
+/**
+ * Default silent threshold
+ * At least one LSB 16-bit data (compare is on absolute value).
+ */
+export const SILENT_THRESHOLD = 1.0 / 32767;
+/**
+ * Default mono detection threshold
+ * Data must be identical within one LSB 16-bit to be identified as mono.
+ */
+export const MONO_THRESHOLD = 1.0 / 65536;
+/**
+ * Default low volume detection threshold
+ */
+export const LOW_VOLUME_THRESHOLD = -60; // dB
+/**
+ * Default clipping detection threshold
+ */
+export const CLIP_THRESHOLD = 0.98;
+/**
+ * Default Voice probability threshold
+ */
+export const VOICE_PROBABILITY_THRESHOLD = 0.3;
+/**
+ * Default clipping count threshold
+ * Number of consecutive clipThreshold level samples that indicate clipping.
+ */
+export const CLIP_COUNT_THRESHOLD = 6.0;
+/**
+ * AudioStats builder
+ *
+ * @param stats - overwrite the default attributes
+ * @param options - `silentThreshold`, `lowVolumeThreshold` and
+ * `clipCountThreshold`
+ */
+export const createAudioStats = (stats = {}, { silentThreshold, lowVolumeThreshold, clipCountThreshold, } = {}) => {
+    return {
+        peak: stats.peak ?? 0,
+        maxRms: stats.maxRms ?? 0,
+        maxClipCount: stats.maxClipCount ?? 0,
+        sumSquare: stats.sumSquare ?? 0,
+        sumLength: stats.sumLength ?? 0,
+        get silent() {
+            return isSilent([this.peak], silentThreshold);
+        },
+        get clipping() {
+            return isClipping(this.maxClipCount, clipCountThreshold);
+        },
+        set clipping(value) {
+            this.clipping = value;
+        },
+        get rms() {
+            return this.sumLength && Math.sqrt(this.sumSquare / this.sumLength);
+        },
+        get lowVolume() {
+            return this.rms === undefined
+                ? false
+                : isLowVolume(this.rms, lowVolumeThreshold);
+        },
+    };
+};
+/**
+ * Convert a byte to float, according to web audio spec
+ *
+ * Floating point audio sample number is defined as: non-interleaved IEEE754
+ * 32-bit linear PCM with a nominal range between -1 and +1, that is, 32bits
+ * floating point buffer, with each samples between -1.0 and 1.0
+ * https://developer.mozilla.org/en-US/docs/Web/API/AudioBuffer
+ *
+ * Byte samples are represented as follows:
+ * 128 is silence, 0 is negative max, 256 is positive max
+ *
+ * @param value - The byte value to convert to float
+ *
+ * @remarks
+ * Ref. https://www.w3.org/TR/webaudio/#dom-analysernode-getbytetimedomaindata
+ */
+export const fromByteToFloat = (value) => (value - 128.0) / 128.0;
+/**
+ * Convert a float to byte, according to web audio spec
+ *
+ * Floating point audio sample number is defined as: non-interleaved IEEE754
+ * 32-bit linear PCM with a nominal range between -1 and +1, that is, 32bits
+ * floating point buffer, with each samples between -1.0 and 1.0
+ * https://developer.mozilla.org/en-US/docs/Web/API/AudioBuffer
+ *
+ * Byte samples are represented as follows:
+ * 128 is silence, 0 is negative max, 256 is positive max
+ *
+ * @param value - The float value to convert to byte
+ *
+ * @remarks
+ * Ref. https://www.w3.org/TR/webaudio/#dom-analysernode-getbytetimedomaindata
+ */
+export const fromFloatToByte = (value) => round(value * 128.0 + 128.0);
+/**
+ * Copy data from Uint8Array buffer to Float32Array buffer with byte to float conversion
+ *
+ * @param bytes - The source Byte buffer
+ * @param floats - The destination buffer
+ */
+export const copyByteBufferToFloatBuffer = (bytes, floats) => {
+    bytes.forEach((value, idx) => {
+        floats[idx] = fromByteToFloat(value);
+    });
+};
+/**
+ * Convert a floating point gain value into a dB representation without any
+ * reference, dBFS, https://en.wikipedia.org/wiki/DBFS
+ *
+ * See https://www.w3.org/TR/webaudio#conversion-to-db
+ *
+ * @param amplitude - Expected a value in (0, 1]
+ */
+export const toDecibel = (gain) => 20 * Math.log10(Math.abs(gain));
+/**
+ * Calculate the averaged volume using Root Mean Square, assuming the data is in
+ * float form
+ *
+ * @param data - Audio Frequency data
+ *
+ * @alpha
+ */
+export const processAverageVolume = (data) => data.length ? rms(data) : 0;
+/**
+ * Simple silent detection to only check the first and last bit from the sample
+ *
+ * @param samples - Audio sample data, this could be in a form of floating number
+ * of a byte number as long as the `threshold` value is given accordingly.
+ * @param threshold - Silent threshold
+ *
+ * @defaultValue
+ * `1.0 / 32767` assuming the sample is float value
+ *
+ * @returns
+ * `true` when it is silent
+ */
+export const isSilent = (samples, threshold = SILENT_THRESHOLD) => samples.length === 0 ||
+    (getFirstSample(samples) <= threshold &&
+        getLastSample(samples) <= threshold);
+function getFirstSample(samples) {
+    if (samples[0] !== undefined) {
+        return Math.abs(samples[0]);
+    }
+    return 0;
+}
+function getLastSample(samples) {
+    const last = samples[samples.length - 1];
+    if (last) {
+        return Math.abs(last);
+    }
+    return 0;
+}
+/**
+ * Check if the provided gain above the low volume threshold, which is
+ * considered as low volume.
+ *
+ * @param gain - Floating point representation of the gain number
+ *
+ * @returns
+ * `true` if the `gain` is lower than the threshold
+ */
+export const isLowVolume = (gain, threshold = LOW_VOLUME_THRESHOLD) => toDecibel(gain) < threshold;
+/**
+ * Check if there is clipping
+ *
+ * @param clipCount - Number of consecutive clip
+ *
+ * @returns
+ * `true` if the `clipCount` is above the threshold, aka clipping
+ */
+export const isClipping = (clipCount, threshold = CLIP_COUNT_THRESHOLD) => clipCount > threshold;
+/**
+ * Check if provided channels are mono or stereo
+ *
+ * @param channels - Audio channels and assuming the inputs are in floating
+ * point form
+ * @param threshold - Mono detection threshold, default to floating point form
+ *
+ * @defaultValue
+ * `1.0 / 32767`
+ *
+ * @returns
+ * `true` if they are mono, otherwise stereo
+ */
+export const isMono = (channels, threshold = MONO_THRESHOLD) => {
+    let sampleDiffCount = 0;
+    if (channels.length < 2 ||
+        channels.filter(channel => !isSilent(channel)).length < 2) {
+        return true;
+    }
+    if (channels[0]?.length === channels[1]?.length) {
+        channels[0]?.forEach((l, idx) => {
+            const r = channels[1]?.[idx];
+            if (r !== undefined && Math.abs(l - r) > threshold) {
+                sampleDiffCount++;
+            }
+        });
+    }
+    else {
+        sampleDiffCount++;
+    }
+    return sampleDiffCount === 0;
+};
+/**
+ * Calculate the audio stats, expected the samples are in float form
+ *
+ * @param options - See StatsOptions
+ *
+ * @remarks
+ * http://www.rossbencina.com/code/real-time-audio-programming-101-time-waits-for-nothing
+ */
+export const getAudioStats = ({ samples, baseStats, clipThreshold = CLIP_THRESHOLD, }) => {
+    let rms = 0;
+    let clipCount = 0;
+    let maxClipCount = 0;
+    let peak = 0;
+    const stats = baseStats || createAudioStats();
+    samples.forEach((s) => {
+        const absS = Math.abs(s);
+        peak = Math.max(peak, absS);
+        if (absS >= clipThreshold) {
+            clipCount += 1;
+            maxClipCount = Math.max(clipCount, maxClipCount);
+        }
+        else {
+            clipCount = 0;
+        }
+        rms += absS * absS;
+    });
+    stats.peak = Math.max(stats.peak ?? 0, peak);
+    stats.sumSquare += rms;
+    stats.sumLength += samples.length;
+    rms = samples.length ? Math.sqrt(rms / samples.length) : 0;
+    stats.maxRms = Math.max(stats.maxRms ?? 0, rms);
+    stats.maxClipCount = Math.max(maxClipCount, stats.maxClipCount ?? 0);
+    return stats;
+};
+/**
+ * A Naive Voice activity detection
+ *
+ * @param options - See `VAOptions`
+ *
+ * @returns `(volume: number) => boolean`, `true` if there is voice
+ */
+export const isVoiceActivity = ({ volumeThreshold = 0.05, VADTimeThreshold = 500, clock = performance, } = {}) => {
+    let lastVADTime = 0;
+    return (volume) => {
+        if (volume >= volumeThreshold) {
+            const now = clock.now();
+            if (!lastVADTime) {
+                lastVADTime = now;
+                return false;
+            }
+            if (now - lastVADTime >= VADTimeThreshold) {
+                return true;
+            }
+            return false;
+        }
+        if (lastVADTime) {
+            lastVADTime = 0;
+        }
+        return false;
+    };
+};
+/**
+ * Compare the provided width and height to see if they are the same
+ *
+ * @param widthA - The width of A
+ * @param heightA - The height of A
+ * @param widthB - The width of B
+ * @param heightB - The height of B
+ */
+export const isEqualSize = (widthA, heightA, widthB, heightB) => widthA === widthB && heightA === heightB;
+/**
+ * Convert the source size to destination size when necessary based on the
+ * height
+ *
+ * @param sw - Source width
+ * @param sh - Source height
+ * @param dw - destination width
+ * @param dh - destination height
+ */
+export const fitDestinationSize = (sw, sh, dw, dh) => {
+    if (!sw || !sh || !dw || !dh) {
+        return { x: 0, y: 0, width: 0, height: 0 };
+    }
+    if (isEqualSize(sw, sh, dw, dh)) {
+        return { x: 0, y: 0, width: sw, height: sh };
+    }
+    const height = Math.floor(dw * (sh / sw));
+    const y = Math.floor((dh - height) / 2);
+    return { x: 0, y, width: dw, height };
+};
+/**
+ * A function to check provided time series data is considered as voice activity
+ *
+ * @param options - @see VAOptions
+ */
+export const createVoiceDetectorFromTimeData = (options = {}) => {
+    const isVoice = isVoiceActivity(options);
+    return timeData => isVoice(rms(timeData));
+};
+/**
+ * A function to check the provided probability is considered as voice activity
+ *
+ * @param voiceThreshold - A threshold of the probability to be considered as
+ * voice activity
+ */
+export const createVoiceDetectorFromProbability = (voiceThreshold = VOICE_PROBABILITY_THRESHOLD) => probability => probability >= voiceThreshold;
+/**
+ * Create a voice detector based on provided params
+ *
+ * @param onDetected - When there is voice activity, this callback will be called
+ * @param shouldDetect - When return `true`, voice activity will function, otherwise, not function
+ * @param options - @see ThrottleOptions
+ */
+export const createVADetector = (onDetected, shouldDetect, options) => (isVoice) => {
+    const throttledTrigger = throttleProcess(onDetected, options?.throttleMs, options?.clock);
+    const process = (data) => {
+        if (!shouldDetect()) {
+            return;
+        }
+        if (isVoice(data)) {
+            throttledTrigger();
+        }
+    };
+    return process;
+};
+/**
+ * Create a function to process the AudioStats and check if silent
+ * `onSignalDetected` callback is called under 2 situations:
+ *
+ * ```
+ * Logic
+ * lastCheck | silent | should call onSignalDetected
+ * 0         | 0      | 0
+ * 0         | 1      | 1
+ * 1         | 0      | 1
+ * 1         | 1      | 0
+ * ```
+ */
+export const createAudioSignalDetector = (shouldDetect, onDetected) => (buffer, threshold) => {
+    const props = { silent: false, lastCheck: false };
+    return (samples) => {
+        if (!shouldDetect()) {
+            buffer.empty();
+            props.silent = false;
+            props.lastCheck = false;
+            return;
+        }
+        if (buffer.enqueue(samples) >= buffer.maxSize) {
+            props.lastCheck = props.silent;
+            props.silent = buffer
+                .dequeueAll()
+                .every(samples => isSilent(samples, threshold));
+            if (props.lastCheck !== props.silent) {
+                onDetected(props.silent);
+            }
+        }
+    };
+};

package/dist/main/processor.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="dom-webcodecs" />
+export declare const createMediaStreamTrackProcessor: {
+    (init: MediaStreamAudioTrackProcessorInit): MediaStreamTrackProcessor<AudioData>;
+    (init: MediaStreamVideoTrackProcessorInit): MediaStreamTrackProcessor<VideoFrame>;
+};

package/dist/main/processor.js

@@ -0,0 +1,4 @@
+export const createMediaStreamTrackProcessor = init => {
+    const processor = new MediaStreamTrackProcessor(init);
+    return processor;
+};

package/dist/main/transformer.d.ts

@@ -0,0 +1 @@
+export declare const createStreamTransformer: <I, O>(transformer?: Transformer<I, O> | undefined, writableStrategy?: QueuingStrategy<I> | undefined, readableStrategy?: QueuingStrategy<O> | undefined) => TransformStream<I, O>;

package/dist/main/transformer.js

@@ -0,0 +1,4 @@
+export const createStreamTransformer = (transformer, writableStrategy, readableStrategy) => {
+    const transformStream = new TransformStream(transformer, writableStrategy, readableStrategy);
+    return transformStream;
+};