@pexip/media 20.1.0 → 20.3.0

package/dist/videoProcessor.js

@@ -1,5 +1,5 @@
-import { createVideoProcessor, createCanvasTransform, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
-import { muteStreamTrack, extractConstraintsWithKeys, getValueFromConstrainNumber, } from '@pexip/media-control';
+import { createCanvasTransform, createVideoProcessor, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
+import { extractConstraintsWithKeys, getValueFromConstrainNumber, muteStreamTrack, } from '@pexip/media-control';
 import { isEmpty, assert } from '@pexip/utils';
 import { createMediaTrack, getBlurKernelSize } from './utils';
 import { logger, proxyWithLog } from './logger';
@@ -94,7 +94,7 @@ export const updateFeatureProps = (constraints, props) => {
         }
     }, {});
 };
-const applyFeatures = (transformer, features) => {
+const applyFeatures = (transformer, features, lowestProcessingHeight) => {
     for (const key in features) {
         const k = key;
         switch (k) {
@@ -110,7 +110,7 @@ const applyFeatures = (transformer, features) => {
             case 'backgroundBlurAmount': {
                 const value = features[k];
                 if (value !== undefined) {
-                    transformer[k] = getBlurKernelSize(value, transformer.height);
+                    transformer[k] = getBlurKernelSize(value, transformer.height, lowestProcessingHeight);
                 }
                 break;
             }
@@ -141,19 +141,15 @@ const getTrackProcessor = (shouldUseStreamTrackProcessor, ...params) => {
     }
     return createVideoTrackProcessorWithFallback(...params);
 };
-export const createVideoStreamProcess = ({ trackProcessorAPI = () => 'stream', processingWidth, processingHeight, shouldEnable, frameRate, 
-//backgroundBlurAmount,
-videoSegmentation, 
-//edgeBlurAmount,
-foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label = PROCESSOR_LABELS.VideoProcessor, dynamicProcessingDimensions = () => false, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
-...options }) => {
+export const createVideoStreamProcess = ({ backgroundImageUrl, dynamicProcessingDimensions = () => false, edgeBlurAmount, foregroundThreshold, frameRate, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
+label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute = () => false, trackProcessorAPI = () => 'stream', videoSegmentation, signals, ...options }) => {
     const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
     const segmenter = options.segmenters[videoSegmentationModel];
     if (!segmenter) {
         throw new Error('Segmenter is undefined');
     }
     const backgroundBlurAmount = options.backgroundBlurAmount &&
-        getBlurKernelSize(options.backgroundBlurAmount, processingHeight);
+        getBlurKernelSize(options.backgroundBlurAmount, processingHeight, lowestProcessingHeight);
     const transformer = options.transformer ??
         createCanvasTransform(segmenter, {
             width: processingWidth,
@@ -199,29 +195,39 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label
         const features = updateFeatureProps(prevMediaTrack.getConstraints(), props);
         const shouldEnabled = shouldEnable();
         if (!shouldEnabled || !prevMediaTrack.track) {
-            logger.debug({ label, features, shouldEnabled }, 'Video processing is skipped');
+            logger.debug({
+                label,
+                features,
+                shouldEnabled,
+            }, 'Video processing is skipped');
             return prevMediaTrack;
         }
         if (!props.hasInitialized) {
             if (dynamicProcessingDimensions()) {
                 const { width, height } = prevMediaTrack.getSettings();
-                /**
-                 * when the processor is closed, the segmenter is not killed, but the track is released.
-                 * this means we can't listen on the track settings changes when we change quality
-                 * we need to update instead of open if the segmenter is already running
-                 */
-                if (props.transformer.segmenter.status === 'new') {
-                    await props.videoProcessor().open({
-                        processingWidth: width,
-                        processingHeight: height,
-                    });
-                    props.transformer.backend = gpuAPI();
-                }
-                else {
-                    props.transformer.update({
-                        processingWidth: width,
-                        processingHeight: height,
-                    });
+                if (width && height) {
+                    /**
+                     * when the processor is closed, the segmenter is not killed, but the track is released.
+                     * this means we can't listen on the track settings changes when we change quality
+                     * we need to update instead of open if the segmenter is already running
+                     */
+                    if (props.transformer.segmenter.status === 'new') {
+                        await props.videoProcessor().open({
+                            processingWidth: width,
+                            processingHeight: height,
+                        });
+                        props.transformer.backend = gpuAPI();
+                    }
+                    else {
+                        props.transformer.update({
+                            processingWidth: width,
+                            processingHeight: height,
+                        });
+                        if (props.backgroundBlurAmount !== undefined) {
+                            props.transformer.backgroundBlurAmount =
+                                getBlurKernelSize(props.backgroundBlurAmount, height, lowestProcessingHeight);
+                        }
+                    }
                 }
             }
             else {
@@ -230,7 +236,7 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label
             }
             props.hasInitialized = true;
         }
-        applyFeatures(props.transformer, features);
+        applyFeatures(props.transformer, features, lowestProcessingHeight);
         const model = features.videoSegmentationModel &&
             props.segmenters[features.videoSegmentationModel];
         if (features.videoSegmentationModel &&
@@ -247,30 +253,61 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label
         assert(track, 'Video track should be there');
         // Inherit contentHint from previous track
         track.contentHint = prevMediaTrack.track.contentHint;
-        const release = async () => {
-            props.videoProcessor().close();
-            await prevMediaTrack.release();
-            _videoProcessor = undefined;
-            props.hasInitialized = false;
-        };
-        const mute = (mute) => {
-            props.transformer.effects = mute
-                ? 'none'
-                : props.videoSegmentation ?? 'none';
-            prevMediaTrack.mute(mute);
-            muteStreamTrack(stream)(mute, 'video');
-        };
+        let trackReleaseTimeoutID = 0;
         return createMediaTrack({
             label,
             kind: 'videoinput',
-            constraints: prevMediaTrack.getConstraints(),
-            previousMediaTrack: prevMediaTrack,
-            input: prevMediaTrack.input,
-            expectedInput: prevMediaTrack.expectedInput,
+            get constraints() {
+                return this.previousMediaTrack?.getConstraints();
+            },
+            get previousMediaTrack() {
+                return prevMediaTrack;
+            },
+            get input() {
+                return this.previousMediaTrack?.input;
+            },
+            get expectedInput() {
+                return this.previousMediaTrack?.expectedInput;
+            },
             track,
-            mute,
-            release,
-            applyConstraints: async (constraints) => {
+            overrideMute: true,
+            get muted() {
+                return this.previousMediaTrack?.muted;
+            },
+            mute(mute, _self, soft) {
+                props.transformer.effects = mute
+                    ? 'none'
+                    : props.videoSegmentation ?? 'none';
+                muteStreamTrack(stream)(mute, 'video');
+                if (stopAsMute() && !soft) {
+                    if (mute) {
+                        // Track cannot be stopped immediately as we need to ensure
+                        // the last frame is black to avoid frozen frame in a gateway call
+                        trackReleaseTimeoutID = window.setTimeout(() => {
+                            for (const track of stream.getVideoTracks()) {
+                                track.stop();
+                            }
+                            void this?.release?.();
+                            trackReleaseTimeoutID = 0;
+                        }, 500);
+                    }
+                    else {
+                        if (trackReleaseTimeoutID) {
+                            window.clearTimeout(trackReleaseTimeoutID);
+                            trackReleaseTimeoutID = 0;
+                        }
+                    }
+                }
+                this.previousMediaTrack?.mute(mute, soft);
+            },
+            signals,
+            async release() {
+                props.videoProcessor().close();
+                await this.previousMediaTrack?.release();
+                _videoProcessor = undefined;
+                props.hasInitialized = false;
+            },
+            async applyConstraints(constraints) {
                 if (isEmpty(constraints)) {
                     return;
                 }
@@ -279,14 +316,18 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label
                 if (isEmpty(features)) {
                     return;
                 }
-                applyFeatures(props.transformer, features);
+                applyFeatures(props.transformer, features, lowestProcessingHeight);
                 if (dynamicProcessingDimensions()) {
-                    const acceptedSettings = prevMediaTrack.source.getSettings();
-                    if (acceptedSettings.width && acceptedSettings.height) {
+                    const acceptedSettings = this.previousMediaTrack?.source.getSettings();
+                    if (acceptedSettings?.width && acceptedSettings?.height) {
                         props.transformer.update({
                             processingHeight: acceptedSettings.height,
                             processingWidth: acceptedSettings.width,
                         });
+                        if (props.backgroundBlurAmount !== undefined) {
+                            props.transformer.backgroundBlurAmount =
+                                getBlurKernelSize(props.backgroundBlurAmount, acceptedSettings.height, lowestProcessingHeight);
+                        }
                     }
                 }
                 const model = features.videoSegmentationModel &&

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "20.1.0",
+  "version": "20.3.0",
   "description": "Home for media related stuff",
   "homepage": "https://gitlab.com/pexip/zoo",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -44,14 +44,14 @@
     "typecheck": "yarn tsc --noEmit -p ."
   },
   "dependencies": {
-    "@pexip/media-control": "20.1.0",
-    "@pexip/media-processor": "20.1.0",
-    "@pexip/signal": "16.8.0",
-    "@pexip/utils": "17.0.0"
+    "@pexip/media-control": "20.3.0",
+    "@pexip/media-processor": "20.3.0",
+    "@pexip/signal": "16.9.0",
+    "@pexip/utils": "17.0.1"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@pexip/bundler": "18.1.0",
+    "@pexip/bundler": "18.1.2",
     "@swc/core": "^1.10.12",
     "@swc/jest": "^0.2.37",
     "jest": "^29.5.12",

package/dist/tsconfig.build.tsbuildinfo

@@ -1 +0,0 @@
-{"root":["../src/audioMixingProcessor.ts","../src/audioProcessor.ts","../src/baseLogger.ts","../src/constants.ts","../src/displayMedia.ts","../src/index.ts","../src/logger.ts","../src/media.ts","../src/previewController.ts","../src/signals.ts","../src/status.ts","../src/typeGuard.ts","../src/types.ts","../src/userMedia.ts","../src/utils.ts","../src/videoProcessor.ts","../../../@types/globals.d.ts"],"version":"5.7.3"}