@pexip/media 17.3.0 → 18.0.0

package/dist/videoProcessor.js

@@ -0,0 +1,362 @@
+import { createVideoProcessor, createCanvasTransform, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
+import { muteStreamTrack, extractConstraintsWithKeys, getValueFromConstrainNumber, } from '@pexip/media-control';
+import { isEmpty } from '@pexip/utils';
+import { shallowCopy, wrapToJSON, applyExtendedConstraints, getBlurKernelSize, } from './utils';
+import { logger, proxyWithLog } from './logger';
+import { isVideoContentHint } from './typeGuard';
+const FEATURE_KEYS = [
+    'backgroundBlurAmount',
+    'backgroundImageUrl',
+    'maskCombineRatio',
+    'edgeBlurAmount',
+    'foregroundThreshold',
+    'frameRate',
+    'videoSegmentation',
+    'videoSegmentationModel',
+    'width',
+    'height',
+    'pan',
+    'tilt',
+    'zoom',
+    'contentHint',
+];
+const getVideoConstraints = extractConstraintsWithKeys(FEATURE_KEYS);
+export const updateFeatureProps = (constraints, props) => {
+    const extracted = getVideoConstraints(constraints);
+    return FEATURE_KEYS.reduce((accm, key) => {
+        switch (key) {
+            case 'contentHint': {
+                const [[feature] = []] = extracted[key];
+                if (isVideoContentHint(feature) && props[key] !== feature) {
+                    props[key] = feature;
+                    return { ...accm, [key]: feature };
+                }
+                return accm;
+            }
+            case 'videoSegmentation': {
+                const [[feature] = []] = extracted[key];
+                if (isRenderEffects(feature) && props[key] !== feature) {
+                    props[key] = feature;
+                    return { ...accm, [key]: feature };
+                }
+                return accm;
+            }
+            case 'videoSegmentationModel': {
+                const [[feature] = []] = extracted[key];
+                if (isSegmentationModel(feature) && props[key] !== feature) {
+                    props[key] = feature;
+                    return { ...accm, [key]: feature };
+                }
+                return accm;
+            }
+            case 'backgroundImageUrl': {
+                const [[feature] = []] = extracted[key];
+                if (feature) {
+                    props[key] = feature;
+                    return { ...accm, [key]: feature };
+                }
+                return accm;
+            }
+            case 'width':
+            case 'height':
+            case 'frameRate':
+            case 'foregroundThreshold':
+            case 'edgeBlurAmount':
+            case 'maskCombineRatio':
+            case 'backgroundBlurAmount': {
+                const [feature] = extracted[key];
+                if (feature !== undefined) {
+                    const value = getValueFromConstrainNumber(feature);
+                    if (props[key] !== value) {
+                        props[key] = value;
+                        return {
+                            ...accm,
+                            [key]: value,
+                        };
+                    }
+                }
+                return accm;
+            }
+            case 'pan':
+            case 'tilt':
+            case 'zoom': {
+                const [feature] = extracted[key];
+                if (feature !== undefined && props[key] !== feature) {
+                    props[key] = feature;
+                    return { ...accm, [key]: feature };
+                }
+                return accm;
+            }
+        }
+    }, {});
+};
+const applyFeatures = (transformer, features) => {
+    Object.keys(features).forEach(key => {
+        const k = key;
+        switch (k) {
+            case 'edgeBlurAmount':
+            case 'maskCombineRatio':
+            case 'foregroundThreshold': {
+                const value = features[k];
+                if (value !== undefined) {
+                    transformer[k] = value;
+                }
+                return;
+            }
+            case 'backgroundBlurAmount': {
+                const value = features[k];
+                if (value !== undefined) {
+                    transformer[k] = getBlurKernelSize(value, transformer.height);
+                }
+                return;
+            }
+            case 'videoSegmentation': {
+                const value = features[k];
+                if (value && value !== transformer.effects) {
+                    transformer.effects = value;
+                }
+                return;
+            }
+            case 'backgroundImageUrl': {
+                const value = features[k];
+                if (value && value !== transformer.backgroundImageUrl) {
+                    transformer.backgroundImageUrl = value;
+                }
+                return;
+            }
+            default: {
+                return;
+            }
+        }
+    });
+};
+const adjustResolution = async (media, features, processingSize) => {
+    if (features.videoSegmentation) {
+        const { video: [videoSettings], } = media.getSettings();
+        const constraints = updateFeatureProps(media.constraints?.video, {});
+        switch (features.videoSegmentation) {
+            case 'blur':
+            case 'overlay': {
+                if (videoSettings?.height !== processingSize.height) {
+                    try {
+                        await media.applyConstraints({
+                            video: {
+                                width: processingSize.width,
+                                height: processingSize.height,
+                            },
+                        });
+                        const { video: [postVideoSettings], } = media.getSettings();
+                        if (postVideoSettings?.height !== processingSize.height) {
+                            // Workaround Firefox 16:9 ratio https://bugzilla.mozilla.org/show_bug.cgi?id=1193640
+                            await media.applyConstraints({
+                                video: { height: 720 },
+                            });
+                        }
+                    }
+                    catch (error) {
+                        // Workaround Firefox 16:9 ratio https://bugzilla.mozilla.org/show_bug.cgi?id=1193640
+                        await media.applyConstraints({
+                            video: { height: 720 },
+                        });
+                    }
+                }
+                break;
+            }
+            case 'none': {
+                if (constraints.height &&
+                    constraints.height !== videoSettings?.height) {
+                    await media.applyConstraints({
+                        video: {
+                            height: constraints.height,
+                        },
+                    });
+                }
+                break;
+            }
+        }
+    }
+};
+const getTrackProcessor = (shouldUseStreamTrackProcessor, ...params) => {
+    if (shouldUseStreamTrackProcessor &&
+        'MediaStreamTrackProcessor' in window) {
+        return createVideoTrackProcessor();
+    }
+    return createVideoTrackProcessorWithFallback(...params);
+};
+export const createVideoStreamProcess = ({ trackProcessorAPI = () => 'stream', processingWidth, processingHeight, shouldEnable, frameRate, 
+//backgroundBlurAmount,
+videoSegmentation, 
+//edgeBlurAmount,
+foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope = 'media', ...options }) => {
+    const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
+    const segmenter = options.segmenters[videoSegmentationModel];
+    if (!segmenter) {
+        throw new Error('Segmenter is undefined');
+    }
+    const backgroundBlurAmount = options.backgroundBlurAmount &&
+        getBlurKernelSize(options.backgroundBlurAmount, processingHeight);
+    const transformer = options.transformer ??
+        createCanvasTransform(segmenter, {
+            width: processingWidth,
+            height: processingHeight,
+            effects: videoSegmentation,
+            foregroundThreshold,
+            backgroundBlurAmount,
+            edgeBlurAmount,
+            backgroundImageUrl,
+            maskCombineRatio,
+        });
+    const proxy = proxyWithLog(logger, scope);
+    let _videoProcessor = undefined;
+    const props = {
+        videoSegmentationModel,
+        segmenters: {
+            selfie: options.segmenters.selfie &&
+                proxy(options.segmenters.selfie, 'Segmenter'),
+            deeplabV3: options.segmenters.deeplabV3 &&
+                proxy(options.segmenters.deeplabV3, 'Segmenter'),
+        },
+        transformer: proxy(transformer, 'Transformer'),
+        videoProcessor: () => {
+            if (!_videoProcessor) {
+                _videoProcessor = proxy(createVideoProcessor([transformer], getTrackProcessor(trackProcessorAPI() === 'stream', {
+                    width: processingWidth,
+                    height: processingHeight,
+                    frameRate,
+                })), 'VideoProcessor');
+            }
+            return _videoProcessor;
+        },
+        videoSegmentation,
+        backgroundBlurAmount,
+        edgeBlurAmount,
+        foregroundThreshold,
+        frameRate,
+        backgroundImageUrl,
+        maskCombineRatio,
+        hasInitialized: options.hasInitializedDeps ?? false,
+    };
+    return async (mediaP) => {
+        const media = await mediaP;
+        const features = updateFeatureProps(media.constraints?.video, props);
+        const shouldEnabled = shouldEnable();
+        if (!shouldEnabled || !media.stream?.getVideoTracks().length) {
+            logger.debug({ scope, features, shouldEnabled }, 'Video processing is skipped');
+            return media;
+        }
+        try {
+            if (!props.hasInitialized) {
+                await props.videoProcessor().open();
+                props.hasInitialized = true;
+            }
+            applyFeatures(props.transformer, features);
+            await adjustResolution(media, features, {
+                width: processingWidth,
+                height: processingHeight,
+            });
+            const model = features.videoSegmentationModel &&
+                props.segmenters[features.videoSegmentationModel];
+            if (features.videoSegmentationModel &&
+                features.videoSegmentationModel !==
+                    props.transformer.segmenter.modelAsset.modelName &&
+                model) {
+                props.transformer.segmenter = model;
+            }
+            const stream = await props.videoProcessor().process(media.stream);
+            const release = async () => {
+                props.videoProcessor().close();
+                await media.release();
+                _videoProcessor = undefined;
+                props.hasInitialized = false;
+            };
+            const muteAudio = (mute) => {
+                media.muteAudio(mute);
+                muteStreamTrack(stream)(mute, 'audio');
+            };
+            const muteVideo = (mute) => {
+                media.muteVideo(mute);
+                props.transformer.effects = mute
+                    ? 'none'
+                    : props.videoSegmentation ?? 'none';
+                muteStreamTrack(stream)(mute, 'video');
+            };
+            const applyConstraints = applyExtendedConstraints(media, async (constraints) => {
+                if (isEmpty(constraints.video)) {
+                    return;
+                }
+                const features = updateFeatureProps(constraints.video, props);
+                logger.debug({ scope, constraints: constraints.video, features }, 'apply video constraints');
+                if (isEmpty(features)) {
+                    return;
+                }
+                try {
+                    applyFeatures(props.transformer, features);
+                    await adjustResolution(media, features, {
+                        width: processingWidth,
+                        height: processingHeight,
+                    });
+                    const model = features.videoSegmentationModel &&
+                        props.segmenters[features.videoSegmentationModel];
+                    if (model &&
+                        features.videoSegmentationModel !==
+                            props.transformer.segmenter.modelAsset.modelName) {
+                        props.transformer.segmenter = model;
+                    }
+                }
+                catch (error) {
+                    if (error instanceof Error) {
+                        logger.error({
+                            scope,
+                            constraints: constraints.video,
+                            features,
+                            error,
+                        }, 'failed to apply video constraints');
+                        options.onError?.(error);
+                    }
+                }
+            });
+            const prevGetSettings = media.getSettings;
+            return wrapToJSON(shallowCopy(media, {
+                stream,
+                muteAudio,
+                muteVideo,
+                applyConstraints,
+                release,
+                getSettings: () => {
+                    const { audio, video } = prevGetSettings();
+                    const contentHint = stream.getAudioTracks().at(0)
+                        ?.contentHint ?? '';
+                    const videoSettings = {
+                        videoSegmentation: transformer.effects,
+                        foregroundThreshold: transformer.foregroundThreshold,
+                        // Background blur amount is a function of height
+                        backgroundBlurAmount: props.backgroundBlurAmount,
+                        edgeBlurAmount: transformer.edgeBlurAmount,
+                        maskCombineRatio: transformer.maskCombineRatio,
+                        backgroundImageUrl: transformer.backgroundImageUrl,
+                        videoSegmentationModel: transformer.segmenter.modelAsset.modelName,
+                        pan: props.pan,
+                        tilt: props.tilt,
+                        zoom: props.zoom,
+                        contentHint,
+                    };
+                    const settings = {
+                        audio,
+                        video: video.map(settings => ({
+                            ...settings,
+                            ...videoSettings,
+                        })),
+                    };
+                    logger.debug({ scope, settings: videoSettings }, 'get video processor settings');
+                    return settings;
+                },
+            }));
+        }
+        catch (e) {
+            if (e instanceof Error) {
+                options.onError?.(e);
+            }
+            return media;
+        }
+    };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "17.3.0",
+  "version": "18.0.0",
   "description": "Home for media related stuff",
   "homepage": "https://gitlab.com/pexip/zoo",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -11,7 +11,7 @@
   },
   "license": "Apache-2.0",
   "author": "Terrence Lam <terrence@tlam.dev>",
-  "module": "dist/index.mjs",
+  "module": "dist/index.js",
   "types": "dist/index.d.ts",
   "directories": {
     "lib": "lib",
@@ -32,15 +32,15 @@
     "typecheck": "yarn tsc --noEmit -p ."
   },
   "dependencies": {
-    "@pexip/media-control": "17.3.0",
-    "@pexip/media-processor": "17.3.0",
-    "@pexip/signal": "16.6.0",
-    "@pexip/utils": "16.9.0"
+    "@pexip/media-control": "18.0.0",
+    "@pexip/media-processor": "18.0.0",
+    "@pexip/signal": "16.7.0",
+    "@pexip/utils": "16.11.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@pexip/bundler": "16.5.0",
-    "prettier": "^3.0.3",
+    "@pexip/bundler": "16.6.1",
+    "prettier": "^3.2.5",
     "typescript": "~5.3.0"
   },
   "publishConfig": {