@pexip/media-processor 21.0.0 → 22.0.0

package/dist/main/video/video.js

@@ -1,51 +1,346 @@
-import { stopStreamTracks } from '../utils';
-import { AbortReason } from './constants';
-export const createVideoProcessor = (transformers, processTrack) => {
+import { assert, isEmpty } from '@pexip/utils';
+import { AbortReason } from '../../common/constants';
+import { BACKGROUND_BLUR_AMOUNT, BACKGROUND_THRESHOLD, DOWN_SAMPLE_FACTOR, EDGE_BLUR_AMOUNT, EXCLUDE_BYSTANDERS, FOREGROUND_THRESHOLD, LIGHT_WRAP_BLUR_AMOUNT, LIGHT_WRAP_EDGE_BAND, LIGHT_WRAP_INTENSITY, LIGHT_WRAP_TIGHTNESS, MASK_COMBINE_RATIO, MORPH_DILATE_RADIUS_PX, MORPH_ERODE_RADIUS_PX, MORPH_PASS, PERSON_CENTER, PROCESSING_HEIGHT, PROCESSING_WIDTH, SIGMA_RANGE, SIGMA_SPACE, } from '../../common/constants';
+import { clamping, createObjectUpdater, compareImageRecords, } from '../../common/utils';
+import { assertMediaStreamVideoTrack, isRenderEffects } from './typeGuards';
+const clampInRangeFrom0To1 = clamping(0, 1);
+const clampInRangeFrom0To9 = clamping(0, 9);
+const createRenderOptions = ({ foregroundThreshold = FOREGROUND_THRESHOLD, backgroundThreshold = BACKGROUND_THRESHOLD, backgroundBlurAmount = BACKGROUND_BLUR_AMOUNT, edgeBlurAmount = EDGE_BLUR_AMOUNT, maskCombineRatio = MASK_COMBINE_RATIO, backgroundImageUrl = '', sigmaSpace = SIGMA_SPACE, sigmaRangeLo = SIGMA_RANGE[0], sigmaRangeHi = SIGMA_RANGE[1], effects = 'none', backend = 'webgl', personCenter = PERSON_CENTER, excludeBystanders = EXCLUDE_BYSTANDERS, morphPass = MORPH_PASS, morphErodeRadiusPx = MORPH_ERODE_RADIUS_PX, morphDilateRadiusPx = MORPH_DILATE_RADIUS_PX, lightWrapIntensity = LIGHT_WRAP_INTENSITY, downSampleFactor = DOWN_SAMPLE_FACTOR, lightWrapTightness = LIGHT_WRAP_TIGHTNESS, lightWrapBlurAmount = LIGHT_WRAP_BLUR_AMOUNT, lightWrapEdgeBand = LIGHT_WRAP_EDGE_BAND, } = {}, onChange) => {
+    const props = {
+        foregroundThreshold,
+        backgroundThreshold,
+        backgroundBlurAmount,
+        edgeBlurAmount,
+        maskCombineRatio,
+        backgroundImageUrl,
+        sigmaSpace,
+        sigmaRangeLo,
+        sigmaRangeHi,
+        effects,
+        backend,
+        personCenter,
+        excludeBystanders,
+        morphPass,
+        morphErodeRadiusPx,
+        morphDilateRadiusPx,
+        lightWrapIntensity,
+        downSampleFactor,
+        lightWrapTightness,
+        lightWrapBlurAmount,
+        lightWrapEdgeBand,
+    };
+    return {
+        original: props,
+        proxy: new Proxy(props, {
+            get(target, p, receiver) {
+                return Reflect.get(target, p, receiver);
+            },
+            set(target, p, newValue, receiver) {
+                switch (p) {
+                    case 'backgroundThreshold': {
+                        const value = clamping(0, target.foregroundThreshold)(newValue);
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'foregroundThreshold': {
+                        const value = clamping(target.backgroundThreshold, 1)(newValue);
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'maskCombineRatio': {
+                        const value = clampInRangeFrom0To1(newValue);
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'lightWrapBlurAmount':
+                    case 'backgroundBlurAmount':
+                    case 'edgeBlurAmount': {
+                        // Expecting integer number
+                        const value = Math.trunc(clampInRangeFrom0To9(newValue));
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'backgroundImageUrl': {
+                        if (newValue !== target[p]) {
+                            Reflect.set(target, p, newValue, receiver);
+                            onChange(p, newValue);
+                        }
+                        break;
+                    }
+                    case 'effects': {
+                        if (isRenderEffects(newValue) &&
+                            newValue !== target[p]) {
+                            Reflect.set(target, p, newValue, receiver);
+                            onChange(p, newValue);
+                        }
+                        break;
+                    }
+                    case 'backend': {
+                        if ((newValue === 'webgl' || newValue === 'webgpu') &&
+                            newValue !== target[p]) {
+                            Reflect.set(target, p, newValue, receiver);
+                            onChange(p, newValue);
+                        }
+                        break;
+                    }
+                    case 'sigmaSpace': {
+                        // less than 0 is unknown behavior, more than 12 is too much
+                        const value = clamping(0, 12)(newValue);
+                        if (value !== target.sigmaSpace) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'sigmaRangeLo': {
+                        const value = clamping(0, target.sigmaRangeHi)(newValue);
+                        if (value !== target.sigmaRangeLo) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'sigmaRangeHi': {
+                        const value = clamping(target.sigmaRangeLo, 1)(newValue);
+                        if (value !== target.sigmaRangeHi) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'personCenter': {
+                        if (Array.isArray(newValue)) {
+                            const [x, y] = newValue.map(clampInRangeFrom0To1);
+                            if (x !== undefined &&
+                                y !== undefined &&
+                                (x !== target.personCenter[0] ||
+                                    y !== target.personCenter[1])) {
+                                Reflect.set(target, p, [x, y], receiver);
+                                onChange(p, [x, y]);
+                            }
+                        }
+                        break;
+                    }
+                    case 'excludeBystanders': {
+                        if (newValue !== target[p]) {
+                            Reflect.set(target, p, newValue, receiver);
+                            onChange(p, newValue);
+                        }
+                        break;
+                    }
+                    case 'morphDilateRadiusPx':
+                    case 'morphErodeRadiusPx': {
+                        const value = clamping(0, 12)(newValue);
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'morphPass': {
+                        // Expecting integer number
+                        const value = Math.trunc(clamping(0, 10)(newValue));
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'lightWrapIntensity': {
+                        const value = clampInRangeFrom0To1(newValue);
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'lightWrapTightness': {
+                        // According to the doc
+                        const value = clamping(0.5, 4)(newValue);
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'lightWrapEdgeBand': {
+                        const value = clamping(0, (target.foregroundThreshold -
+                            target.backgroundThreshold) /
+                            2)(newValue);
+                        if (value !== target[p]) {
+                            Reflect.set(target, p, value, receiver);
+                            onChange(p, value);
+                        }
+                        break;
+                    }
+                    case 'downSampleFactor': {
+                        // NOTE: There is no impl other than 2 and 4, so we just hardcode it here
+                        if ((newValue === 2 || newValue === 4) &&
+                            newValue !== target[p]) {
+                            Reflect.set(target, p, newValue, receiver);
+                            onChange(p, newValue);
+                        }
+                        break;
+                    }
+                    default: {
+                        // Catch all the unknown keys, and just ignore the set operation
+                        return false;
+                    }
+                }
+                // No throw
+                return true;
+            },
+        }),
+    };
+};
+export const createVideoProcessor = (segmenter, inputProcessAdaptor) => {
+    const canUpdate = () => segmenter.status !== 'new';
     const props = {
         processing: false,
     };
+    let changedRenderOptions = {};
+    const initRenderOptions = () => createRenderOptions({}, (k, v) => {
+        changedRenderOptions[k] = v;
+    });
+    let renderOptions = initRenderOptions();
+    let processorOptions = {
+        renderOptions: renderOptions.original,
+        processingWidth: PROCESSING_WIDTH,
+        processingHeight: PROCESSING_HEIGHT,
+    };
+    const updateRendererOptions = createObjectUpdater(renderOptions.proxy);
+    const updateChangedRendererOptions = (changed) => {
+        for (const [k, v] of Object.entries(changed)) {
+            //@ts-expect-error
+            processorOptions.renderOptions[k] = v;
+        }
+    };
+    const updateProcessorOptions = async (options) => {
+        const _processorOptions = {};
+        if (options?.renderOptions) {
+            updateRendererOptions(options.renderOptions);
+            if (!isEmpty(changedRenderOptions)) {
+                changedRenderOptions = {};
+                _processorOptions.renderOptions =
+                    processorOptions.renderOptions;
+            }
+        }
+        if (options?.backgroundImage &&
+            !compareImageRecords(options.backgroundImage, processorOptions.backgroundImage)) {
+            processorOptions.backgroundImage = options.backgroundImage;
+            _processorOptions.backgroundImage =
+                processorOptions.backgroundImage;
+        }
+        if (options.processingWidth) {
+            processorOptions.processingWidth = options.processingWidth;
+            _processorOptions.processingWidth =
+                processorOptions.processingWidth;
+        }
+        if (options.processingHeight) {
+            processorOptions.processingHeight = options.processingHeight;
+            _processorOptions.processingHeight =
+                processorOptions.processingHeight;
+        }
+        if (options.imageSegmenterOptions) {
+            processorOptions.imageSegmenterOptions = {
+                ...processorOptions.imageSegmenterOptions,
+                ...options.imageSegmenterOptions,
+            };
+            _processorOptions.imageSegmenterOptions =
+                processorOptions.imageSegmenterOptions;
+        }
+        if (canUpdate() && !isEmpty(_processorOptions)) {
+            const changed = await segmenter.update(_processorOptions);
+            updateChangedRendererOptions(changed);
+        }
+    };
     const open = async (options) => {
-        await Promise.all(transformers.map(transformer => transformer.init(options)));
+        if (options) {
+            await updateProcessorOptions(options);
+        }
+        if (segmenter.status === 'new') {
+            const changed = await segmenter.open(processorOptions);
+            updateChangedRendererOptions(changed);
+        }
+    };
+    const update = async (options) => {
+        await updateProcessorOptions(options);
     };
     const process = async (source) => {
-        const [track] = source.getVideoTracks();
-        if (!track) {
-            return source;
-        }
         if (props.processing) {
-            throw new Error('Cannot process when it is already processing');
-        }
-        props.abortController = new AbortController();
-        const trackGenerated = await processTrack(track, transformers, {
-            signal: props.abortController.signal,
-        });
-        props.outputStream = new MediaStream([
-            trackGenerated,
-            ...source.getAudioTracks().map(track => track.clone()),
-        ]);
+            close();
+        }
+        assertMediaStreamVideoTrack(source);
+        if (segmenter.status === 'new') {
+            await segmenter.open(processorOptions);
+            // @ts-expect-error -- Segmenter status could be changed to Error if anything went wrong
+            if (segmenter.status === 'Error') {
+                return source;
+            }
+        }
+        const processed = await segmenter.process(inputProcessAdaptor.adapt(source), processorOptions.renderOptions);
+        props.outputTrack = inputProcessAdaptor.outputTrack;
+        if (processed instanceof MediaStreamTrack) {
+            props.outputTrack = processed;
+        }
+        else if (!props.outputTrack) {
+            props.outputTrack = source;
+        }
+        assert(props.outputTrack, 'Segmenter failed to generate output track');
         props.processing = true;
-        return props.outputStream;
+        return props.outputTrack;
     };
     const close = () => {
-        if (!props.processing) {
-            return;
-        }
+        segmenter.close();
         props.abortController?.abort(AbortReason.Close);
-        stopStreamTracks(props.outputStream);
-        props.outputStream = undefined;
+        props.outputTrack?.stop();
+        props.outputTrack = undefined;
         props.processing = false;
-        for (const transformer of transformers) {
-            transformer.close();
-        }
     };
     const destroy = async () => {
         close();
-        await Promise.all(transformers.map(transformer => transformer.destroy()));
+        processorOptions?.backgroundImage?.image?.close();
+        renderOptions = initRenderOptions();
+        changedRenderOptions = {};
+        processorOptions = {
+            renderOptions: renderOptions.original,
+            processingWidth: PROCESSING_WIDTH,
+            processingHeight: PROCESSING_HEIGHT,
+        };
+        await segmenter.destroy();
     };
     return {
         open,
+        update,
         process,
+        mute(muted) {
+            segmenter.mute(muted);
+        },
         close,
         destroy,
+        get status() {
+            return segmenter.status;
+        },
+        get renderOptions() {
+            return processorOptions.renderOptions;
+        },
+        get processingWidth() {
+            return processorOptions.processingWidth;
+        },
+        get processingHeight() {
+            return processorOptions.processingHeight;
+        },
     };
 };