@pexip/media 20.3.5 → 22.0.0

package/dist/videoProcessor.js

@@ -1,132 +1,283 @@
-import { createCanvasTransform, createVideoProcessor, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
-import { extractConstraintsWithKeys, getValueFromConstrainNumber, muteStreamTrack, } from '@pexip/media-control';
-import { isEmpty, assert } from '@pexip/utils';
-import { createMediaTrack, getBlurKernelSize } from './utils';
+import { isRenderEffects } from '@pexip/media-processor';
+import { extractConstraints, getValueFromConstrainNumber, } from '@pexip/media-control';
+import { hasOwn, isEmpty } from '@pexip/utils';
+import { createMediaTrack, getBlurKernelSize, getTentBlurSize } from './utils';
 import { logger, proxyWithLog } from './logger';
 import { isVideoContentHint } from './typeGuard';
 import { PROCESSOR_LABELS } from './constants';
-const FEATURE_KEYS = [
-    'backgroundBlurAmount',
-    'backgroundImageUrl',
-    'maskCombineRatio',
-    'edgeBlurAmount',
-    'foregroundThreshold',
-    'frameRate',
-    'videoSegmentation',
-    'videoSegmentationModel',
-    'width',
-    'height',
-    'pan',
-    'tilt',
-    'zoom',
-    'contentHint',
-];
-const getVideoConstraints = extractConstraintsWithKeys(FEATURE_KEYS);
+const FEATURE_KEYS = {
+    backgroundBlurAmount: 'number',
+    backgroundImageUrl: 'string',
+    backgroundThreshold: 'number',
+    maskCombineRatio: 'number',
+    edgeBlurAmount: 'number',
+    foregroundThreshold: 'number',
+    frameRate: 'number',
+    videoSegmentation: 'string',
+    width: 'number',
+    height: 'number',
+    pan: 'boolean',
+    tilt: 'boolean',
+    zoom: 'boolean',
+    contentHint: 'string',
+    sigmaSpace: 'number',
+    sigmaRangeLo: 'number',
+    sigmaRangeHi: 'number',
+    excludeBystanders: 'boolean',
+    personCenterX: 'number',
+    personCenterY: 'number',
+    morphErodeRadiusPx: 'number',
+    morphPass: 'number',
+    morphDilateRadiusPx: 'number',
+    lightWrapIntensity: 'number',
+    lightWrapTightness: 'number',
+    lightWrapEdgeBand: 'number',
+    lightWrapBlurAmount: 'number',
+    downSampleFactor: 'number',
+};
 export const updateFeatureProps = (constraints, props) => {
-    const extracted = getVideoConstraints(constraints);
-    return FEATURE_KEYS.reduce((accm, key) => {
+    const features = {};
+    if (constraints === undefined || typeof constraints === 'boolean') {
+        return features;
+    }
+    for (const k of Object.keys(constraints)) {
+        const key = k;
+        if (!FEATURE_KEYS[key]) {
+            continue;
+        }
         switch (key) {
             case 'contentHint': {
-                const [[feature = ''] = []] = extracted[key];
+                const [[feature = ''] = []] = extractConstraints(key)(constraints);
                 if (isVideoContentHint(feature) && props[key] !== feature) {
                     props[key] = feature;
-                    accm[key] = feature;
-                    return accm;
+                    features[key] = feature;
                 }
-                return accm;
+                break;
             }
             case 'videoSegmentation': {
-                const [[feature] = []] = extracted[key];
+                const [[feature] = []] = extractConstraints(key)(constraints);
                 if (isRenderEffects(feature) && props[key] !== feature) {
                     props[key] = feature;
-                    accm[key] = feature;
-                    return accm;
-                }
-                return accm;
-            }
-            case 'videoSegmentationModel': {
-                const [[feature] = []] = extracted[key];
-                if (isSegmentationModel(feature) && props[key] !== feature) {
-                    props[key] = feature;
-                    accm[key] = feature;
-                    return accm;
+                    features[key] = feature;
                 }
-                return accm;
+                break;
             }
             case 'backgroundImageUrl': {
-                const [[feature] = []] = extracted[key];
+                const [[feature] = []] = extractConstraints(key)(constraints);
                 if (feature) {
                     props[key] = feature;
-                    accm[key] = feature;
-                    return accm;
+                    features[key] = feature;
                 }
-                return accm;
+                break;
             }
-            case 'width':
-            case 'height':
-            case 'frameRate':
-            case 'foregroundThreshold':
+            case 'backgroundBlurAmount':
             case 'edgeBlurAmount':
+            case 'lightWrapBlurAmount': {
+                const [feature] = extractConstraints(key)(constraints);
+                if (feature !== undefined) {
+                    const value = getValueFromConstrainNumber(feature);
+                    if (props[key] !== value) {
+                        props[key] = value;
+                        features[key] = value;
+                    }
+                }
+                break;
+            }
+            case 'backgroundThreshold':
+            case 'downSampleFactor':
+            case 'foregroundThreshold':
+            case 'frameRate':
+            case 'height':
+            case 'lightWrapEdgeBand':
+            case 'lightWrapIntensity':
+            case 'lightWrapTightness':
             case 'maskCombineRatio':
-            case 'backgroundBlurAmount': {
-                const [feature] = extracted[key];
+            case 'morphDilateRadiusPx':
+            case 'morphErodeRadiusPx':
+            case 'morphPass':
+            case 'personCenterX':
+            case 'personCenterY':
+            case 'sigmaRangeHi':
+            case 'sigmaRangeLo':
+            case 'sigmaSpace':
+            case 'width': {
+                const [feature] = extractConstraints(key)(constraints);
                 if (feature !== undefined) {
                     const value = getValueFromConstrainNumber(feature);
                     if (props[key] !== value) {
                         props[key] = value;
-                        accm[key] = value;
-                        return accm;
+                        features[key] = value;
                     }
                 }
-                return accm;
+                break;
             }
             case 'pan':
             case 'tilt':
             case 'zoom': {
-                const [feature] = extracted[key];
+                const [feature] = extractConstraints(key)(constraints);
                 if (feature !== undefined && props[key] !== feature) {
                     props[key] = feature;
-                    accm[key] = feature;
-                    return accm;
+                    features[key] = feature;
                 }
-                return accm;
+                break;
             }
             default:
-                return accm;
+                break;
         }
-    }, {});
+    }
+    return features;
+};
+export const mapFeatureToSettings = (props) => {
+    return new Proxy({}, {
+        get(_target, p, receiver) {
+            switch (p) {
+                case 'tilt':
+                case 'zoom':
+                case 'pan': {
+                    if (props[p] === undefined || props[p] === false) {
+                        return undefined;
+                    }
+                    return 0;
+                }
+                default: {
+                    return Reflect.get(props, p, receiver);
+                }
+            }
+        },
+        has(_target, p) {
+            return p in props;
+        },
+        ownKeys() {
+            return Reflect.ownKeys(props);
+        },
+        getOwnPropertyDescriptor(_target, p) {
+            if (hasOwn(props, p)) {
+                return {
+                    configurable: true,
+                    enumerable: true,
+                    writable: true,
+                };
+            }
+            return undefined;
+        },
+    });
+};
+export const mapSettingsToFeatures = (settings) => {
+    return new Proxy({}, {
+        get(_target, p, receiver) {
+            switch (p) {
+                case 'tilt':
+                case 'zoom':
+                case 'pan': {
+                    if (settings[p] === undefined) {
+                        return undefined;
+                    }
+                    return true;
+                }
+                default: {
+                    return Reflect.get(settings, p, receiver);
+                }
+            }
+        },
+        has(_target, p) {
+            return p in settings;
+        },
+        ownKeys() {
+            return Reflect.ownKeys(settings);
+        },
+        getOwnPropertyDescriptor(target, p) {
+            if (hasOwn(settings, p)) {
+                return {
+                    configurable: true,
+                    enumerable: true,
+                    writable: true,
+                    value: this.get?.(target, p, undefined),
+                };
+            }
+            return undefined;
+        },
+    });
 };
-const applyFeatures = (transformer, features, lowestProcessingHeight) => {
+const applyFeatures = async (processor, features, referenceProcessingHeight) => {
+    const renderOptions = {};
     for (const key in features) {
         const k = key;
         switch (k) {
-            case 'edgeBlurAmount':
             case 'maskCombineRatio':
-            case 'foregroundThreshold': {
+            case 'foregroundThreshold':
+            case 'sigmaSpace':
+            case 'sigmaRangeLo':
+            case 'sigmaRangeHi':
+            case 'morphErodeRadiusPx':
+            case 'morphPass':
+            case 'morphDilateRadiusPx':
+            case 'lightWrapIntensity':
+            case 'lightWrapTightness':
+            case 'lightWrapEdgeBand':
+            case 'downSampleFactor':
+            case 'backgroundThreshold': {
                 const value = features[k];
                 if (value !== undefined) {
-                    transformer[k] = value;
+                    renderOptions[k] = value;
                 }
                 break;
             }
+            case 'excludeBystanders': {
+                const value = features[k];
+                if (value !== undefined) {
+                    renderOptions[k] = value;
+                }
+                break;
+            }
+            case 'personCenterX': {
+                const value = features[k];
+                if (value !== undefined) {
+                    renderOptions.personCenter = [
+                        value,
+                        renderOptions.personCenter?.[1] ??
+                            processor.renderOptions.personCenter[1],
+                    ];
+                }
+                break;
+            }
+            case 'personCenterY': {
+                const value = features[k];
+                if (value !== undefined) {
+                    renderOptions.personCenter = [
+                        renderOptions.personCenter?.[0] ??
+                            processor.renderOptions.personCenter[0],
+                        value,
+                    ];
+                }
+                break;
+            }
+            case 'lightWrapBlurAmount':
             case 'backgroundBlurAmount': {
                 const value = features[k];
                 if (value !== undefined) {
-                    transformer[k] = getBlurKernelSize(value, transformer.height, lowestProcessingHeight);
+                    renderOptions[k] = getBlurKernelSize(value, referenceProcessingHeight ?? processor.processingHeight, processor.processingHeight);
+                }
+                break;
+            }
+            case 'edgeBlurAmount': {
+                const value = features[k];
+                if (value !== undefined) {
+                    renderOptions[k] = getTentBlurSize(value, referenceProcessingHeight ?? processor.processingHeight, processor.processingHeight);
                 }
                 break;
             }
             case 'videoSegmentation': {
                 const value = features[k];
-                if (value && value !== transformer.effects) {
-                    transformer.effects = value;
+                if (value && value !== processor.renderOptions.effects) {
+                    renderOptions.effects = value;
                 }
                 break;
             }
             case 'backgroundImageUrl': {
                 const value = features[k];
-                if (value && value !== transformer.backgroundImageUrl) {
-                    transformer.backgroundImageUrl = value;
+                if (value &&
+                    value !== processor.renderOptions.backgroundImageUrl) {
+                    renderOptions.backgroundImageUrl = value;
                 }
                 break;
             }
@@ -135,63 +286,40 @@ const applyFeatures = (transformer, features, lowestProcessingHeight) => {
             }
         }
     }
-};
-const getTrackProcessor = (shouldUseStreamTrackProcessor, ...params) => {
-    if (shouldUseStreamTrackProcessor &&
-        'MediaStreamTrackProcessor' in window) {
-        return createVideoTrackProcessor();
+    if (!isEmpty(renderOptions)) {
+        await processor.update({ renderOptions });
     }
-    return createVideoTrackProcessorWithFallback(...params);
 };
-export const createVideoStreamProcess = ({ backgroundImageUrl, dynamicProcessingDimensions = () => false, edgeBlurAmount, foregroundThreshold, frameRate, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
-label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute = () => false, trackProcessorAPI = () => 'stream', videoSegmentation, signals, ...options }) => {
-    const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
-    const segmenter = options.segmenters[videoSegmentationModel];
-    if (!segmenter) {
-        throw new Error('Segmenter is undefined');
-    }
+export const createVideoStreamProcess = ({ backgroundImageUrl, dynamicProcessingDimensions = () => false, backgroundThreshold, downSampleFactor, foregroundThreshold, lightWrapEdgeBand, lightWrapIntensity, lightWrapTightness, morphDilateRadiusPx, morphErodeRadiusPx, morphPass, personCenterX, personCenterY, sigmaSpace, sigmaRangeHi, sigmaRangeLo, frameRate, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
+label = PROCESSOR_LABELS.VideoProcessor, referenceProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute = () => false, videoSegmentation, signals, getVideoProcessor, ...options }) => {
     const backgroundBlurAmount = options.backgroundBlurAmount &&
-        getBlurKernelSize(options.backgroundBlurAmount, processingHeight, lowestProcessingHeight);
-    const transformer = options.transformer ??
-        createCanvasTransform(segmenter, {
-            width: processingWidth,
-            height: processingHeight,
-            effects: videoSegmentation,
-            foregroundThreshold,
-            backgroundBlurAmount,
-            edgeBlurAmount,
-            backgroundImageUrl,
-            maskCombineRatio,
-        });
-    const proxy = proxyWithLog(logger, label);
-    let _videoProcessor = undefined;
+        getBlurKernelSize(options.backgroundBlurAmount, referenceProcessingHeight ?? processingHeight, processingHeight);
+    const lightWrapBlurAmount = options.lightWrapBlurAmount &&
+        getBlurKernelSize(options.lightWrapBlurAmount, referenceProcessingHeight ?? processingHeight, processingHeight);
+    const edgeBlurAmount = options.edgeBlurAmount &&
+        getTentBlurSize(options.edgeBlurAmount, referenceProcessingHeight ?? processingHeight, processingHeight);
     const props = {
-        videoSegmentationModel,
-        segmenters: {
-            selfie: options.segmenters.selfie &&
-                proxy(options.segmenters.selfie, 'Segmenter'),
-            deeplabV3: options.segmenters.deeplabV3 &&
-                proxy(options.segmenters.deeplabV3, 'Segmenter'),
-        },
-        transformer: proxy(transformer, 'Transformer'),
-        videoProcessor: () => {
-            if (!_videoProcessor) {
-                _videoProcessor = proxy(createVideoProcessor([transformer], getTrackProcessor(trackProcessorAPI() === 'stream', {
-                    width: processingWidth,
-                    height: processingHeight,
-                    frameRate,
-                })), 'VideoProcessor');
-            }
-            return _videoProcessor;
-        },
         videoSegmentation,
         backgroundBlurAmount,
+        backgroundImageUrl,
+        backgroundThreshold,
+        downSampleFactor,
         edgeBlurAmount,
         foregroundThreshold,
         frameRate,
-        backgroundImageUrl,
+        lightWrapBlurAmount,
+        lightWrapEdgeBand,
+        lightWrapIntensity,
+        lightWrapTightness,
         maskCombineRatio,
-        hasInitialized: options.hasInitializedDeps ?? false,
+        morphDilateRadiusPx,
+        morphErodeRadiusPx,
+        morphPass,
+        personCenterX,
+        personCenterY,
+        sigmaRangeHi,
+        sigmaRangeLo,
+        sigmaSpace,
     };
     return async (prevMediaTrack) => {
         const features = updateFeatureProps(prevMediaTrack.getConstraints(), props);
@@ -204,55 +332,33 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
             }, 'Video processing is skipped');
             return prevMediaTrack;
         }
-        if (!props.hasInitialized) {
-            if (dynamicProcessingDimensions()) {
-                const { width, height } = prevMediaTrack.getSettings();
-                if (width && height) {
-                    /**
-                     * when the processor is closed, the segmenter is not killed, but the track is released.
-                     * this means we can't listen on the track settings changes when we change quality
-                     * we need to update instead of open if the segmenter is already running
-                     */
-                    if (props.transformer.segmenter.status === 'new') {
-                        await props.videoProcessor().open({
-                            processingWidth: width,
-                            processingHeight: height,
-                        });
-                        props.transformer.backend = gpuAPI();
-                    }
-                    else {
-                        props.transformer.update({
-                            processingWidth: width,
-                            processingHeight: height,
-                        });
-                        if (props.backgroundBlurAmount !== undefined) {
-                            props.transformer.backgroundBlurAmount =
-                                getBlurKernelSize(props.backgroundBlurAmount, height, lowestProcessingHeight);
-                        }
-                    }
-                }
+        const { videoSegmentation, ...renderOptions } = props;
+        const videoProcessor = proxyWithLog(logger, label)(getVideoProcessor(), 'VideoProcessor');
+        if (dynamicProcessingDimensions()) {
+            const { width, height } = prevMediaTrack.getSettings();
+            if (props.backgroundBlurAmount !== undefined) {
+                renderOptions.backgroundBlurAmount = getBlurKernelSize(props.backgroundBlurAmount, referenceProcessingHeight ?? processingHeight, height ?? processingHeight);
+            }
+            if (props.lightWrapBlurAmount !== undefined) {
+                renderOptions.lightWrapBlurAmount = getBlurKernelSize(props.lightWrapBlurAmount, referenceProcessingHeight ?? processingHeight, height ?? processingHeight);
             }
-            else {
-                await props.videoProcessor().open();
-                props.transformer.backend = gpuAPI();
+            if (props.edgeBlurAmount !== undefined) {
+                renderOptions.edgeBlurAmount = getTentBlurSize(props.edgeBlurAmount, referenceProcessingHeight ?? processingHeight, height ?? processingHeight);
             }
-            props.hasInitialized = true;
+            await videoProcessor.update({
+                processingWidth: width,
+                processingHeight: height,
+                renderOptions: {
+                    backend: gpuAPI(),
+                    ...renderOptions,
+                    effects: videoSegmentation,
+                },
+            });
         }
-        applyFeatures(props.transformer, features, lowestProcessingHeight);
-        const model = features.videoSegmentationModel &&
-            props.segmenters[features.videoSegmentationModel];
-        if (features.videoSegmentationModel &&
-            features.videoSegmentationModel !==
-                props.transformer.segmenter.modelAsset.modelName &&
-            model) {
-            props.transformer.segmenter = model;
+        else {
+            await applyFeatures(videoProcessor, { ...features, videoSegmentation }, referenceProcessingHeight);
         }
-        const stream = await props
-            .videoProcessor()
-            // FIXME: Change process to accept MediaStreamTrack instead of MediaStream
-            .process(new MediaStream([prevMediaTrack.track]));
-        const track = stream.getVideoTracks().at(0);
-        assert(track, 'Video track should be there');
+        const track = await videoProcessor.process(prevMediaTrack.track);
         // Inherit contentHint from previous track
         track.contentHint = prevMediaTrack.track.contentHint;
         let trackReleaseTimeoutID = 0;
@@ -277,18 +383,15 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
                 return this.previousMediaTrack?.muted;
             },
             mute(mute, _self, soft) {
-                props.transformer.effects = mute
-                    ? 'none'
-                    : props.videoSegmentation ?? 'none';
-                muteStreamTrack(stream)(mute, 'video');
+                videoProcessor.mute(mute);
+                track.enabled = !mute;
                 if (stopAsMute() && !soft) {
                     if (mute) {
                         // Track cannot be stopped immediately as we need to ensure
                         // the last frame is black to avoid frozen frame in a gateway call
                         trackReleaseTimeoutID = window.setTimeout(() => {
-                            for (const track of stream.getVideoTracks()) {
-                                track.stop();
-                            }
+                            track.stop();
+                            videoProcessor.close();
                             void this?.release?.();
                             trackReleaseTimeoutID = 0;
                         }, 500);
@@ -304,10 +407,8 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
             },
             signals,
             async release() {
-                props.videoProcessor().close();
+                videoProcessor.close();
                 await this.previousMediaTrack?.release();
-                _videoProcessor = undefined;
-                props.hasInitialized = false;
             },
             async applyConstraints(constraints) {
                 if (isEmpty(constraints)) {
@@ -318,43 +419,64 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
                 if (isEmpty(features)) {
                     return;
                 }
-                applyFeatures(props.transformer, features, lowestProcessingHeight);
+                await applyFeatures(videoProcessor, features, referenceProcessingHeight);
                 if (dynamicProcessingDimensions()) {
                     const acceptedSettings = this.previousMediaTrack?.source.getSettings();
                     if (acceptedSettings?.width && acceptedSettings?.height) {
-                        props.transformer.update({
+                        const renderOptions = {};
+                        if (props.backgroundBlurAmount !== undefined) {
+                            renderOptions.backgroundBlurAmount =
+                                getBlurKernelSize(props.backgroundBlurAmount, referenceProcessingHeight ??
+                                    processingHeight, acceptedSettings.height);
+                        }
+                        if (props.lightWrapBlurAmount !== undefined) {
+                            renderOptions.lightWrapBlurAmount =
+                                getBlurKernelSize(props.lightWrapBlurAmount, referenceProcessingHeight ??
+                                    processingHeight, acceptedSettings.height);
+                        }
+                        if (props.edgeBlurAmount !== undefined) {
+                            renderOptions.edgeBlurAmount = getTentBlurSize(props.edgeBlurAmount, referenceProcessingHeight ?? processingHeight, acceptedSettings.height);
+                        }
+                        await videoProcessor.update({
                             processingHeight: acceptedSettings.height,
                             processingWidth: acceptedSettings.width,
+                            renderOptions,
                         });
-                        if (props.backgroundBlurAmount !== undefined) {
-                            props.transformer.backgroundBlurAmount =
-                                getBlurKernelSize(props.backgroundBlurAmount, acceptedSettings.height, lowestProcessingHeight);
-                        }
                     }
                 }
-                const model = features.videoSegmentationModel &&
-                    props.segmenters[features.videoSegmentationModel];
-                if (model &&
-                    features.videoSegmentationModel !==
-                        props.transformer.segmenter.modelAsset.modelName) {
-                    props.transformer.segmenter = model;
-                }
             },
             getSettings: () => {
                 const contentHint = track.contentHint ?? '';
+                const ro = videoProcessor.renderOptions;
                 return {
-                    videoSegmentation: transformer.effects,
-                    foregroundThreshold: transformer.foregroundThreshold,
-                    // Background blur amount is a function of height
                     backgroundBlurAmount: props.backgroundBlurAmount,
-                    edgeBlurAmount: transformer.edgeBlurAmount,
-                    maskCombineRatio: transformer.maskCombineRatio,
-                    backgroundImageUrl: transformer.backgroundImageUrl,
-                    videoSegmentationModel: transformer.segmenter.modelAsset.modelName,
-                    pan: props.pan,
-                    tilt: props.tilt,
-                    zoom: props.zoom,
+                    backgroundImageUrl: ro.backgroundImageUrl,
+                    backgroundThreshold: ro.backgroundThreshold,
                     contentHint,
+                    downSampleFactor: ro.downSampleFactor,
+                    edgeBlurAmount: ro.edgeBlurAmount,
+                    excludeBystanders: ro.excludeBystanders,
+                    foregroundThreshold: ro.foregroundThreshold,
+                    frameRate: props.frameRate,
+                    height: props.height,
+                    lightWrapBlurAmount: ro.lightWrapBlurAmount,
+                    lightWrapEdgeBand: ro.lightWrapEdgeBand,
+                    lightWrapIntensity: ro.lightWrapIntensity,
+                    lightWrapTightness: ro.lightWrapTightness,
+                    maskCombineRatio: ro.maskCombineRatio,
+                    morphDilateRadiusPx: ro.morphDilateRadiusPx,
+                    morphErodeRadiusPx: ro.morphErodeRadiusPx,
+                    morphPass: ro.morphPass,
+                    panEnabled: props.pan,
+                    personCenterX: ro.personCenter[0],
+                    personCenterY: ro.personCenter[1],
+                    sigmaRangeHi: ro.sigmaRangeHi,
+                    sigmaRangeLo: ro.sigmaRangeLo,
+                    sigmaSpace: ro.sigmaSpace,
+                    tiltEnabled: props.tilt,
+                    videoSegmentation: ro.effects,
+                    width: props.width,
+                    zoomEnabled: props.zoom,
                 };
             },
         });