@abrar71/lib-jitsi-meet 0.0.0

package/dist/esm/modules/detection/VADNoiseDetection.js

@@ -0,0 +1,168 @@
+import EventEmitter from '../util/EventEmitter';
+import { calculateAverage, filterPositiveValues } from '../util/MathUtil';
+import { DetectionEvents } from './DetectionEvents';
+/**
+ * The average value VAD needs to be under over a period of time to be considered noise.
+ * @type {number}
+ */
+const VAD_NOISE_AVG_THRESHOLD = 0.2;
+/**
+ * The average values that audio input need to be over to be considered loud.
+ * @type {number}
+ */
+const NOISY_AUDIO_LEVEL_THRESHOLD = 0.040;
+/**
+ * The value that a VAD score needs to be under in order for processing to begin.
+ * @type {number}
+ */
+const VAD_SCORE_TRIGGER = 0.2;
+/**
+ * The value that a VAD score needs to be under in order for processing to begin.
+ * @type {number}
+ */
+const AUDIO_LEVEL_SCORE_TRIGGER = 0.020;
+/**
+ * Time span over which we calculate an average score used to determine if we trigger the event.
+ * @type {number}
+ */
+const PROCESS_TIME_FRAME_SPAN_MS = 1500;
+/**
+ * Detect if provided VAD score and PCM data is considered noise.
+ */
+export default class VADNoiseDetection extends EventEmitter {
+    /**
+     * Creates <tt>VADNoiseDetection</tt>
+     *
+     * @constructor
+     */
+    constructor() {
+        super();
+        /**
+         * Flag which denotes the current state of the detection service i.e.if there is already a processing operation
+         * ongoing.
+         */
+        this._processing = false;
+        /**
+         * Buffer that keeps the VAD scores for a period of time.
+         */
+        this._scoreArray = [];
+        /**
+         * Buffer that keeps audio level samples for a period of time.
+         */
+        this._audioLvlArray = [];
+        /**
+         * Current state of the service, if it's not active no processing will occur.
+         */
+        this._active = false;
+        /**
+         * Timeout reference for processing VAD scores.
+         */
+        this._processTimeout = null;
+        this._calculateNoisyScore = this._calculateNoisyScore.bind(this);
+    }
+    /**
+     * Compute cumulative VAD score and PCM audio levels once the PROCESS_TIME_FRAME_SPAN_MS timeout has elapsed.
+     * If the score is above the set threshold fire the event.
+     * @returns {void}
+     * @fires VAD_NOISY_DEVICE
+     */
+    _calculateNoisyScore() {
+        const scoreAvg = calculateAverage(new Float32Array(this._scoreArray));
+        const audioLevelAvg = calculateAverage(new Float32Array(this._audioLvlArray));
+        if (scoreAvg < VAD_NOISE_AVG_THRESHOLD && audioLevelAvg > NOISY_AUDIO_LEVEL_THRESHOLD) {
+            this.emit(DetectionEvents.VAD_NOISY_DEVICE);
+            this._setActiveState(false);
+        }
+        // We reset the context in case a new process phase needs to be triggered.
+        this.reset();
+    }
+    /**
+     * Set the active state of the detection service and notify any listeners.
+     *
+     * @param {boolean} active
+     * @fires DETECTOR_STATE_CHANGE
+     */
+    _setActiveState(active) {
+        this._active = active;
+        this.emit(DetectionEvents.DETECTOR_STATE_CHANGE, this._active);
+    }
+    /**
+     * Record the vad score and average volume in the appropriate buffers.
+     *
+     * @param {number} vadScore
+     * @param {number} avgAudioLvl - average audio level of the PCM sample associated with the VAD score.s
+     */
+    _recordValues(vadScore, avgAudioLvl) {
+        this._scoreArray.push(vadScore);
+        this._audioLvlArray.push(avgAudioLvl);
+    }
+    /**
+     * Change the state according to the muted status of the tracked device.
+     *
+     * @param {boolean} isMuted - Is the device muted or not.
+     */
+    changeMuteState(isMuted) {
+        // This service only needs to run when the microphone is not muted.
+        this._setActiveState(!isMuted);
+        this.reset();
+    }
+    /**
+     * Check whether or not the service is active or not.
+     *
+     * @returns {boolean}
+     */
+    isActive() {
+        return this._active;
+    }
+    /**
+     * Listens for {@link TrackVADEmitter} events and processes them.
+     *
+     * @param {Object} vadScore -VAD score emitted by {@link TrackVADEmitter}
+     * @param {number}   vadScore.timestamp - Exact time at which processed PCM sample was generated.
+     * @param {number} vadScore.score - VAD score on a scale from 0 to 1 (i.e. 0.7)
+     * @param {Float32Array} vadScore.pcmData - Raw PCM Data associated with the VAD score.
+     * @param {string} vadScore.deviceId - Device id of the associated track.
+     * @listens VAD_SCORE_PUBLISHED
+     */
+    processVADScore(vadScore) {
+        if (!this._active) {
+            return;
+        }
+        // There is a processing phase on going, add score to buffer array.
+        if (this._processing) {
+            // Filter and calculate sample average so we don't have to process one large array at a time.
+            const pcmDataArray = Array.isArray(vadScore.pcmData) ? new Float32Array(vadScore.pcmData) : vadScore.pcmData;
+            const posAudioLevels = filterPositiveValues(pcmDataArray);
+            this._recordValues(vadScore.score, calculateAverage(new Float32Array(posAudioLevels)));
+            return;
+        }
+        // If the VAD score for the sample is low and audio level has a high enough level we can start listening for
+        // noise
+        if (vadScore.score < VAD_SCORE_TRIGGER) {
+            const pcmDataArray = Array.isArray(vadScore.pcmData) ? new Float32Array(vadScore.pcmData) : vadScore.pcmData;
+            const posAudioLevels = filterPositiveValues(pcmDataArray);
+            const avgAudioLvl = calculateAverage(new Float32Array(posAudioLevels));
+            if (avgAudioLvl > AUDIO_LEVEL_SCORE_TRIGGER) {
+                this._processing = true;
+                this._recordValues(vadScore.score, avgAudioLvl);
+                // Once the preset timeout executes the final score will be calculated.
+                this._processTimeout = setTimeout(this._calculateNoisyScore, PROCESS_TIME_FRAME_SPAN_MS);
+            }
+        }
+    }
+    /**
+     * Reset the processing context, clear buffers, cancel the timeout trigger.
+     *
+     * @returns {void}
+     */
+    reset() {
+        this._processing = false;
+        this._scoreArray = [];
+        this._audioLvlArray = [];
+        if (this._processTimeout) {
+            clearTimeout(this._processTimeout);
+            this._processTimeout = null;
+        }
+    }
+}
+//# sourceMappingURL=VADNoiseDetection.js.map

package/dist/esm/modules/detection/VADNoiseDetection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VADNoiseDetection.js","sourceRoot":"","sources":["../../../../modules/detection/VADNoiseDetection.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE1E,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAGpD;;;GAGG;AACH,MAAM,uBAAuB,GAAW,GAAG,CAAC;AAE5C;;;GAGG;AACH,MAAM,2BAA2B,GAAW,KAAK,CAAC;AAElD;;;GAGG;AACH,MAAM,iBAAiB,GAAW,GAAG,CAAC;AAEtC;;;GAGG;AACH,MAAM,yBAAyB,GAAW,KAAK,CAAC;AAEhD;;;GAGG;AACH,MAAM,0BAA0B,GAAW,IAAI,CAAC;AAEhD;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,iBAAkB,SAAQ,YAAY;IA2BvD;;;;OAIG;IACH;QACI,KAAK,EAAE,CAAC;QAER;;;WAGG;QACH,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QAEtB;;WAEG;QACH,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB;;WAEG;QACH,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAE5B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;;;OAKG;IACK,oBAAoB;QACxB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QACtE,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QAE9E,IAAI,QAAQ,GAAG,uBAAuB,IAAI,aAAa,GAAG,2BAA2B,EAAE,CAAC;YACpF,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAE5C,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,0EAA0E;QAC1E,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,MAAe;QACnC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,qBAAqB,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACnE,CAAC;IAED;;;;;OAKG;IACK,aAAa,CAAC,QAAgB,EAAE,WAAmB;QACvD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACI,eAAe,CAAC,OAAgB;QACnC,mEAAmE;QACnE,IAAI,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACI,QAAQ;QACX,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;;;;;;;OASG;IACI,eAAe,CAAC,QAAmB;QACtC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QAED,mEAAmE;QACnE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACvB,6FAA6F;YACzF,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC7G,MAAM,cAAc,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAE1D,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;YAEvF,OAAO;QACX,CAAC;QAED,4GAA4G;QAC5G,QAAQ;QACR,IAAI,QAAQ,CAAC,KAAK,GAAG,iBAAiB,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC7G,MAAM,cAAc,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC1D,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC,CAAC;YAEvE,IAAI,WAAW,GAAG,yBAAyB,EAAE,CAAC;gBAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;gBAEhD,uEAAuE;gBACvE,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,IAAI,CAAC,oBAAoB,EAAE,0BAA0B,CAAC,CAAC;YAC7F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;OAIG;IACI,KAAK;QACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACnC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAChC,CAAC;IACL,CAAC;CACJ"}

package/dist/esm/modules/detection/VADReportingService.js

@@ -0,0 +1,203 @@
+import { getLogger } from '@jitsi/logger';
+import EventEmitter from '../util/EventEmitter';
+import { DetectionEvents } from './DetectionEvents';
+import TrackVADEmitter from './TrackVADEmitter';
+const logger = getLogger('vad:VADReportingService');
+/**
+ * Sample rate used by TrackVADEmitter, this value determines how often the ScriptProcessorNode is going to call the
+ * process audio function and with what sample size.
+ * Basically lower values mean more callbacks with lower processing times bigger values less callbacks with longer
+ * processing times. This value is somewhere in the middle, so we strike a balance between flooding with callbacks
+ * and processing time. Possible values  256, 512, 1024, 2048, 4096, 8192, 16384. Passing other values will default
+ * to closes neighbor.
+ */
+const SCRIPT_NODE_SAMPLE_RATE = 4096;
+/**
+ * Voice activity detection reporting service. The service create TrackVADEmitters for the provided devices and
+ * publishes an average of their VAD score over the specified interval via EventEmitter.
+ * The service is not reusable if destroyed a new one needs to be created, i.e. when a new device is added to the system
+ * a new service needs to be created and the old discarded.
+ */
+export default class VADReportingService extends EventEmitter {
+    /**
+     *
+     * @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices.
+     *
+     * @constructor
+     */
+    constructor(intervalDelay) {
+        super();
+        /**
+         * Map containing context for devices currently being monitored by the reporting service.
+         */
+        this._contextMap = new Map();
+        /**
+         * State flag, check if the instance was destroyed.
+         */
+        this._destroyed = false;
+        /**
+         * Delay at which to publish VAD score for monitored devices.
+         */
+        this._intervalDelay = intervalDelay;
+        /**
+         * Identifier for the interval publishing stats on the set interval.
+         */
+        this._intervalId = null;
+        logger.debug(`Constructed VADReportingService with publish interval of: ${intervalDelay}`);
+    }
+    /**
+     * Factory methods that creates the TrackVADEmitters for the associated array of devices and instantiates
+     * a VADReportingService.
+     *
+     * @param {Array<MediaDeviceInfo>} micDeviceList - Device list that is monitored inside the service.
+     * @param {number} intervalDelay - Delay at which to publish VAD score for monitored devices.
+     * @param {Object} createVADProcessor - Function that creates a Voice activity detection processor. The processor
+     * needs to implement the following functions:
+     * - <tt>getSampleLength()</tt> - Returns the sample size accepted by getSampleLength.
+     * - <tt>getRequiredPCMFrequency()</tt> - Returns the PCM frequency at which the processor operates.
+     * - <tt>calculateAudioFrameVAD(pcmSample)</tt> - Process a 32 float pcm sample of getSampleLength size.
+     *
+     * @returns {Promise<VADReportingService>}
+     */
+    static create(micDeviceList, intervalDelay, createVADProcessor) {
+        const vadReportingService = new VADReportingService(intervalDelay);
+        const emitterPromiseArray = [];
+        const audioDeviceList = micDeviceList.filter(device => device.kind === 'audioinput');
+        // Create a TrackVADEmitter for each provided audio input device.
+        for (const micDevice of audioDeviceList) {
+            logger.debug(`Initializing VAD context for mic: ${micDevice.label} -> ${micDevice.deviceId}`);
+            const emitterPromise = createVADProcessor()
+                .then(rnnoiseProcessor => TrackVADEmitter.create(micDevice.deviceId, SCRIPT_NODE_SAMPLE_RATE, rnnoiseProcessor))
+                .then(emitter => {
+                emitter.on(DetectionEvents.VAD_SCORE_PUBLISHED, vadReportingService._devicePublishVADScore.bind(vadReportingService));
+                emitter.start();
+                return {
+                    deviceInfo: micDevice,
+                    scoreArray: [],
+                    vadEmitter: emitter
+                };
+            });
+            emitterPromiseArray.push(emitterPromise);
+        }
+        // Once all the TrackVADEmitter promises are resolved get the ones that were successfully initialized and start
+        // monitoring them.
+        return Promise.allSettled(emitterPromiseArray).then(outcomeArray => {
+            const successfulPromises = outcomeArray.filter(p => p.status === 'fulfilled');
+            const rejectedPromises = outcomeArray.filter(p => p.status === 'rejected');
+            const availableDeviceContexts = successfulPromises.map(p => p.value);
+            const rejectReasons = rejectedPromises.map(p => p.reason);
+            for (const reason of rejectReasons) {
+                logger.error('Failed to acquire audio device with error: ', reason);
+            }
+            vadReportingService._setVADContextArray(availableDeviceContexts);
+            vadReportingService._startPublish();
+            return vadReportingService;
+        });
+    }
+    /**
+     * Destroy TrackVADEmitters and clear the context map.
+     *
+     * @returns {void}
+     */
+    _clearContextMap() {
+        for (const vadContext of this._contextMap.values()) {
+            vadContext.vadEmitter.destroy();
+        }
+        this._contextMap.clear();
+    }
+    /**
+     * Set the watched device contexts.
+     *
+     * @param {Array<IVADDeviceContext>} vadContextArray - List of mics.
+     * @returns {void}
+     */
+    _setVADContextArray(vadContextArray) {
+        for (const vadContext of vadContextArray) {
+            this._contextMap.set(vadContext.deviceInfo.deviceId, vadContext);
+        }
+    }
+    /**
+     * Start the setInterval reporting process.
+     *
+     * @returns {void}.
+     */
+    _startPublish() {
+        logger.debug('VADReportingService started publishing.');
+        this._intervalId = setInterval(() => {
+            this._reportVadScore();
+        }, this._intervalDelay);
+    }
+    /**
+     * Function called at set interval with selected compute. The result will be published on the set callback.
+     *
+     * @returns {void}
+     * @fires VAD_REPORT_PUBLISHED
+     */
+    _reportVadScore() {
+        const vadComputeScoreArray = [];
+        const computeTimestamp = Date.now();
+        // Go through each device and compute cumulated VAD score.
+        for (const [deviceId, vadContext] of this._contextMap) {
+            const nrOfVADScores = vadContext.scoreArray.length;
+            let vadSum = 0;
+            vadContext.scoreArray.forEach(vadScore => {
+                vadSum += vadScore.score;
+            });
+            // TODO For now we just calculate the average score for each device, more compute algorithms will be added.
+            const avgVAD = vadSum / nrOfVADScores;
+            vadContext.scoreArray = [];
+            vadComputeScoreArray.push({
+                deviceId,
+                score: avgVAD,
+                timestamp: computeTimestamp
+            });
+        }
+        logger.debug('VADReportingService reported.', vadComputeScoreArray);
+        /**
+         * Once the computation for all the tracked devices is done, fire an event containing all the necessary
+         * information.
+         *
+         * @event VAD_REPORT_PUBLISHED
+         * @type Array<Object> with the following structure:
+         * @property {Date} timestamp - Timestamo at which the compute took place.
+         * @property {number} avgVAD - Average VAD score over monitored period of time.
+         * @property {string} deviceId - Associate local audio device ID.
+         */
+        this.emit(DetectionEvents.VAD_REPORT_PUBLISHED, vadComputeScoreArray);
+    }
+    /**
+     * Callback method passed to vad emitters in order to publish their score.
+     *
+     * @param {Object} vadScore -VAD score emitted by.
+     * @param {Date}   vadScore.timestamp - Exact time at which processed PCM sample was generated.
+     * @param {number} vadScore.score - VAD score on a scale from 0 to 1 (i.e. 0.7).
+     * @param {string} vadScore.deviceId - Device id of the associated track.
+     * @returns {void}
+     * @listens VAD_SCORE_PUBLISHED
+     */
+    _devicePublishVADScore(vadScore) {
+        const context = this._contextMap.get(vadScore.deviceId);
+        if (context) {
+            context.scoreArray.push(vadScore);
+        }
+    }
+    /**
+     * Destroy the VADReportingService, stops the setInterval reporting, destroys the emitters and clears the map.
+     * After this call the instance is no longer usable.
+     *
+     * @returns {void}.
+     */
+    destroy() {
+        if (this._destroyed) {
+            return;
+        }
+        logger.debug('Destroying VADReportingService.');
+        if (this._intervalId) {
+            clearInterval(this._intervalId);
+            this._intervalId = null;
+        }
+        this._clearContextMap();
+        this._destroyed = true;
+    }
+}
+//# sourceMappingURL=VADReportingService.js.map

package/dist/esm/modules/detection/VADReportingService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VADReportingService.js","sourceRoot":"","sources":["../../../../modules/detection/VADReportingService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE1C,OAAO,YAAY,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,eAAe,MAAM,mBAAmB,CAAC;AAEhD,MAAM,MAAM,GAAG,SAAS,CAAC,yBAAyB,CAAC,CAAC;AAEpD;;;;;;;GAOG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAcrC;;;;;GAKG;AACH,MAAM,CAAC,OAAO,OAAO,mBAAoB,SAAQ,YAAY;IAMzD;;;;;OAKG;IACH,YAAY,aAAqB;QAC7B,KAAK,EAAE,CAAC;QAER;;WAEG;QACH,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7B;;WAEG;QACH,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB;;WAEG;QACH,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QAEpC;;WAEG;QACH,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,MAAM,CAAC,KAAK,CAAC,6DAA6D,aAAa,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,MAAM,CACL,aAAgC,EAChC,aAAqB,EACrB,kBAIE;QAEN,MAAM,mBAAmB,GAAG,IAAI,mBAAmB,CAAC,aAAa,CAAC,CAAC;QACnE,MAAM,mBAAmB,GAAiC,EAAE,CAAC;QAE7D,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;QAErF,iEAAiE;QACjE,KAAK,MAAM,SAAS,IAAI,eAAe,EAAE,CAAC;YACtC,MAAM,CAAC,KAAK,CAAC,qCAAqC,SAAS,CAAC,KAAK,OAAO,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE9F,MAAM,cAAc,GAAG,kBAAkB,EAAE;iBAC1C,IAAI,CAAC,gBAAgB,CAAC,EAAE,CACrB,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;iBACzF,IAAI,CAAC,OAAO,CAAC,EAAE;gBACZ,OAAO,CAAC,EAAE,CACN,eAAe,CAAC,mBAAmB,EACnC,mBAAmB,CAAC,sBAAsB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CACvE,CAAC;gBACF,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,OAAO;oBACH,UAAU,EAAE,SAAS;oBACrB,UAAU,EAAE,EAAE;oBACd,UAAU,EAAE,OAAO;iBACtB,CAAC;YACN,CAAC,CAAC,CAAC;YAEH,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7C,CAAC;QAED,+GAA+G;QAC/G,mBAAmB;QACnB,OAAO,OAAO,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE;YAC/D,MAAM,kBAAkB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;YAC9E,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;YAE3E,MAAM,uBAAuB,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,CAA+C,CAAC,KAAK,CAAC,CAAC;YACpH,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,CAA2B,CAAC,MAAM,CAAC,CAAC;YAErF,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE,MAAM,CAAC,CAAC;YACxE,CAAC;YAED,mBAAmB,CAAC,mBAAmB,CAAC,uBAAuB,CAAC,CAAC;YACjE,mBAAmB,CAAC,aAAa,EAAE,CAAC;YAEpC,OAAO,mBAAmB,CAAC;QAC/B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACZ,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,mBAAmB,CAAC,eAAoC;QACpD,KAAK,MAAM,UAAU,IAAI,eAAe,EAAE,CAAC;YACvC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACrE,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,aAAa;QACT,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,eAAe,EAAE,CAAC;QAC3B,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAC5B,CAAC;IAED;;;;;OAKG;IACH,eAAe;QACX,MAAM,oBAAoB,GAAgB,EAAE,CAAC;QAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEpC,0DAA0D;QAE1D,KAAK,MAAM,CAAE,QAAQ,EAAE,UAAU,CAAE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtD,MAAM,aAAa,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC;YACnD,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE;gBACrC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,CAAC;YAEH,2GAA2G;YAC3G,MAAM,MAAM,GAAG,MAAM,GAAG,aAAa,CAAC;YAEtC,UAAU,CAAC,UAAU,GAAG,EAAE,CAAC;YAE3B,oBAAoB,CAAC,IAAI,CAAC;gBACtB,QAAQ;gBACR,KAAK,EAAE,MAAM;gBACb,SAAS,EAAE,gBAAgB;aAC9B,CAAC,CAAC;QACP,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,oBAAoB,CAAC,CAAC;QAEpE;;;;;;;;;WASG;QACH,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;;;;;;;OASG;IACH,sBAAsB,CAAC,QAAmB;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,OAAO,EAAE,CAAC;YACV,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,OAAO;QACH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,OAAO;QACX,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAEhD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IAC3B,CAAC;CACJ"}

package/dist/esm/modules/detection/VADTalkMutedDetection.js

@@ -0,0 +1,131 @@
+import EventEmitter from '../util/EventEmitter';
+import { calculateAverage } from '../util/MathUtil';
+import { DetectionEvents } from './DetectionEvents';
+/**
+ * The threshold which the average VAD values for a span of time needs to exceed to trigger an event.
+ * @type {number}
+ */
+const VAD_AVG_THRESHOLD = 0.6;
+/**
+ * The VAD score needed to trigger the processing algorithm, i.e. if a sample has the VAD score >= VAD_VOICE_LEVEL
+ * we start processing all scores for a time span defined by const PROCESS_TIME_FRAME_SPAN_MS.
+ * @type {number}
+ */
+const VAD_VOICE_LEVEL = 0.9;
+/**
+ * Sample rate of TrackVADEmitter, it defines how many audio samples are processed at a time.
+ * @type {number}
+ */
+/**
+ * Time span over which we calculate an average score used to determine if we trigger the event.
+ * @type {number}
+ */
+const PROCESS_TIME_FRAME_SPAN_MS = 700;
+/**
+ * Detect if provided VAD score which is generated on a muted device is voice and fires an event.
+ */
+export default class VADTalkMutedDetection extends EventEmitter {
+    /**
+     * Creates <tt>VADTalkMutedDetection</tt>
+     * @constructor
+     */
+    constructor() {
+        super();
+        /**
+         * Flag which denotes the current state of the detection service i.e.if there is already a processing operation
+         * ongoing.
+         */
+        this._processing = false;
+        /**
+         * Buffer that keeps the VAD scores for a period of time.
+         */
+        this._scoreArray = [];
+        /**
+         * Current mute state of the audio track being monitored.
+         */
+        this._active = false;
+        this._calculateVADScore = this._calculateVADScore.bind(this);
+    }
+    /**
+     * Compute cumulative VAD score function called once the PROCESS_TIME_FRAME_SPAN_MS timeout has elapsed.
+     * @returns {void}
+     * @fires VAD_TALK_WHILE_MUTED
+     */
+    _calculateVADScore() {
+        const score = calculateAverage(new Float32Array(this._scoreArray));
+        if (score > VAD_AVG_THRESHOLD) {
+            this.emit(DetectionEvents.VAD_TALK_WHILE_MUTED);
+            // Event was fired. Stop event emitter and remove listeners so no residue events kick off after this point
+            // and a single VAD_TALK_WHILE_MUTED is generated per mic muted state.
+            this._setActiveState(false);
+        }
+        // We reset the context in case a new process phase needs to be triggered.
+        this.reset();
+    }
+    /**
+     * Set the active state of the detection service and notify any listeners.
+     *
+     * @param {boolean} active
+     * @fires DETECTOR_STATE_CHANGE
+     */
+    _setActiveState(active) {
+        this._active = active;
+        this.emit(DetectionEvents.DETECTOR_STATE_CHANGE, this._active);
+    }
+    /**
+     * Change the state according to the muted status of the tracked device.
+     *
+     * @param {boolean} isMuted - Is the device muted or not.
+     */
+    changeMuteState(isMuted) {
+        // This service only needs to run when the microphone is muted.
+        this._setActiveState(isMuted);
+        this.reset();
+    }
+    /**
+     * Check whether or not the service is active or not.
+     *
+     * @returns {boolean}
+     */
+    isActive() {
+        return this._active;
+    }
+    /**
+     * Listens for {@link TrackVADEmitter} events and processes them.
+     *
+     * @param {Object} vadScore -VAD score emitted by {@link TrackVADEmitter}
+     * @param {number}   vadScore.timestamp - Exact time at which processed PCM sample was generated.
+     * @param {number} vadScore.score - VAD score on a scale from 0 to 1 (i.e. 0.7)
+     * @param {string} vadScore.deviceId - Device id of the associated track.
+     * @listens VAD_SCORE_PUBLISHED
+     */
+    processVADScore(vadScore) {
+        if (!this._active) {
+            return;
+        }
+        // There is a processing phase on going, add score to buffer array.
+        if (this._processing) {
+            this._scoreArray.push(vadScore.score);
+            return;
+        }
+        // Because we remove all listeners on the vadEmitter once the main event is triggered,
+        // there is no need to check for rogue events.
+        if (vadScore.score > VAD_VOICE_LEVEL) {
+            this._processing = true;
+            this._scoreArray.push(vadScore.score);
+            // Start gathering VAD scores for the configured period of time.
+            this._processTimeout = setTimeout(this._calculateVADScore, PROCESS_TIME_FRAME_SPAN_MS);
+        }
+    }
+    /**
+     * Reset the processing context, clear buffer, cancel the timeout trigger.
+     *
+     * @returns {void}
+     */
+    reset() {
+        this._processing = false;
+        this._scoreArray = [];
+        clearTimeout(this._processTimeout);
+    }
+}
+//# sourceMappingURL=VADTalkMutedDetection.js.map

package/dist/esm/modules/detection/VADTalkMutedDetection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VADTalkMutedDetection.js","sourceRoot":"","sources":["../../../../modules/detection/VADTalkMutedDetection.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;GAGG;AACH,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;;GAIG;AACH,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;GAGG;AAEH;;;GAGG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAQvC;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,YAAY;IAM3D;;;OAGG;IACH;QACI,KAAK,EAAE,CAAC;QAER;;;WAGG;QACH,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB;;WAEG;QACH,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QAEtB;;WAEG;QACH,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,CAAC;IAED;;;;OAIG;IACK,kBAAkB;QACtB,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAEnE,IAAI,KAAK,GAAG,iBAAiB,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAC;YAEhD,0GAA0G;YAC1G,sEAAsE;YACtE,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,0EAA0E;QAC1E,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,MAAe;QACnC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,qBAAqB,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACnE,CAAC;IAED;;;;OAIG;IACI,eAAe,CAAC,OAAgB;QACnC,+DAA+D;QAC/D,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACI,QAAQ;QACX,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;;;;;;OAQG;IACI,eAAe,CAAC,QAAmB;QACtC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,mEAAmE;QAGnE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAEtC,OAAO;QACX,CAAC;QAED,sFAAsF;QACtF,8CAA8C;QAC9C,IAAI,QAAQ,CAAC,KAAK,GAAG,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAEtC,gEAAgE;YAChE,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,0BAA0B,CAAC,CAAC;QAC3F,CAAC;IACL,CAAC;IAED;;;;OAIG;IACI,KAAK;QACR,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACvC,CAAC;CACJ"}