@pexip/media 20.3.4 → 21.0.0

package/dist/videoProcessor.js

@@ -1,13 +1,14 @@
 import { createCanvasTransform, createVideoProcessor, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
 import { extractConstraintsWithKeys, getValueFromConstrainNumber, muteStreamTrack, } from '@pexip/media-control';
 import { isEmpty, assert } from '@pexip/utils';
-import { createMediaTrack, getBlurKernelSize } from './utils';
+import { createMediaTrack, getBlurKernelSize, getTentBlurSize } from './utils';
 import { logger, proxyWithLog } from './logger';
 import { isVideoContentHint } from './typeGuard';
 import { PROCESSOR_LABELS } from './constants';
 const FEATURE_KEYS = [
     'backgroundBlurAmount',
     'backgroundImageUrl',
+    'backgroundThreshold',
     'maskCombineRatio',
     'edgeBlurAmount',
     'foregroundThreshold',
@@ -20,6 +21,20 @@ const FEATURE_KEYS = [
     'tilt',
     'zoom',
     'contentHint',
+    'sigmaSpace',
+    'sigmaRangeLo',
+    'sigmaRangeHi',
+    'excludeBystanders',
+    'personCenterX',
+    'personCenterY',
+    'morphErodeRadiusPx',
+    'morphPass',
+    'morphDilateRadiusPx',
+    'lightWrapIntensity',
+    'lightWrapTightness',
+    'lightWrapEdgeBand',
+    'lightWrapBlurAmount',
+    'downSampleFactor',
 ];
 const getVideoConstraints = extractConstraintsWithKeys(FEATURE_KEYS);
 export const updateFeatureProps = (constraints, props) => {
@@ -62,13 +77,27 @@ export const updateFeatureProps = (constraints, props) => {
                 }
                 return accm;
             }
-            case 'width':
-            case 'height':
-            case 'frameRate':
-            case 'foregroundThreshold':
+            case 'backgroundBlurAmount':
+            case 'backgroundThreshold':
+            case 'downSampleFactor':
             case 'edgeBlurAmount':
+            case 'foregroundThreshold':
+            case 'frameRate':
+            case 'height':
+            case 'lightWrapBlurAmount':
+            case 'lightWrapEdgeBand':
+            case 'lightWrapIntensity':
+            case 'lightWrapTightness':
             case 'maskCombineRatio':
-            case 'backgroundBlurAmount': {
+            case 'morphDilateRadiusPx':
+            case 'morphErodeRadiusPx':
+            case 'morphPass':
+            case 'personCenterX':
+            case 'personCenterY':
+            case 'sigmaRangeHi':
+            case 'sigmaRangeLo':
+            case 'sigmaSpace':
+            case 'width': {
                 const [feature] = extracted[key];
                 if (feature !== undefined) {
                     const value = getValueFromConstrainNumber(feature);
@@ -96,23 +125,68 @@ export const updateFeatureProps = (constraints, props) => {
         }
     }, {});
 };
-const applyFeatures = (transformer, features, lowestProcessingHeight) => {
+const applyFeatures = (transformer, features, referenceProcessingHeight) => {
     for (const key in features) {
         const k = key;
         switch (k) {
-            case 'edgeBlurAmount':
             case 'maskCombineRatio':
-            case 'foregroundThreshold': {
+            case 'foregroundThreshold':
+            case 'sigmaSpace':
+            case 'sigmaRangeLo':
+            case 'sigmaRangeHi':
+            case 'morphErodeRadiusPx':
+            case 'morphPass':
+            case 'morphDilateRadiusPx':
+            case 'lightWrapIntensity':
+            case 'lightWrapTightness':
+            case 'lightWrapEdgeBand':
+            case 'downSampleFactor':
+            case 'backgroundThreshold': {
+                const value = features[k];
+                if (value !== undefined) {
+                    transformer[k] = value;
+                }
+                break;
+            }
+            case 'excludeBystanders': {
                 const value = features[k];
                 if (value !== undefined) {
                     transformer[k] = value;
                 }
                 break;
             }
+            case 'personCenterX': {
+                const value = features[k];
+                if (value !== undefined) {
+                    transformer.personCenter = [
+                        value,
+                        transformer.personCenter[1],
+                    ];
+                }
+                break;
+            }
+            case 'personCenterY': {
+                const value = features[k];
+                if (value !== undefined) {
+                    transformer.personCenter = [
+                        transformer.personCenter[0],
+                        value,
+                    ];
+                }
+                break;
+            }
+            case 'lightWrapBlurAmount':
             case 'backgroundBlurAmount': {
                 const value = features[k];
                 if (value !== undefined) {
-                    transformer[k] = getBlurKernelSize(value, transformer.height, lowestProcessingHeight);
+                    transformer[k] = getBlurKernelSize(value, referenceProcessingHeight ?? transformer.height, transformer.height);
+                }
+                break;
+            }
+            case 'edgeBlurAmount': {
+                const value = features[k];
+                if (value !== undefined) {
+                    transformer[k] = getTentBlurSize(value, referenceProcessingHeight ?? transformer.height, transformer.height);
                 }
                 break;
             }
@@ -143,28 +217,47 @@ const getTrackProcessor = (shouldUseStreamTrackProcessor, ...params) => {
     }
     return createVideoTrackProcessorWithFallback(...params);
 };
-export const createVideoStreamProcess = ({ backgroundImageUrl, dynamicProcessingDimensions = () => false, edgeBlurAmount, foregroundThreshold, frameRate, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
-label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute = () => false, trackProcessorAPI = () => 'stream', videoSegmentation, signals, ...options }) => {
+export const createVideoStreamProcess = ({ backgroundImageUrl, dynamicProcessingDimensions = () => false, backgroundThreshold, downSampleFactor, foregroundThreshold, lightWrapEdgeBand, lightWrapIntensity, lightWrapTightness, morphDilateRadiusPx, morphErodeRadiusPx, morphPass, personCenterX, personCenterY, sigmaSpace, sigmaRangeHi, sigmaRangeLo, frameRate, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
+label = PROCESSOR_LABELS.VideoProcessor, referenceProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute = () => false, trackProcessorAPI = () => 'stream', videoSegmentation, signals, ...options }) => {
     const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
     const segmenter = options.segmenters[videoSegmentationModel];
     if (!segmenter) {
         throw new Error('Segmenter is undefined');
     }
     const backgroundBlurAmount = options.backgroundBlurAmount &&
-        getBlurKernelSize(options.backgroundBlurAmount, processingHeight, lowestProcessingHeight);
+        getBlurKernelSize(options.backgroundBlurAmount, referenceProcessingHeight ?? processingHeight, processingHeight);
+    const lightWrapBlurAmount = options.lightWrapBlurAmount &&
+        getBlurKernelSize(options.lightWrapBlurAmount, referenceProcessingHeight ?? processingHeight, processingHeight);
+    const edgeBlurAmount = options.edgeBlurAmount &&
+        getTentBlurSize(options.edgeBlurAmount, referenceProcessingHeight ?? processingHeight, processingHeight);
     const transformer = options.transformer ??
         createCanvasTransform(segmenter, {
-            width: processingWidth,
-            height: processingHeight,
-            effects: videoSegmentation,
-            foregroundThreshold,
             backgroundBlurAmount,
-            edgeBlurAmount,
             backgroundImageUrl,
+            backgroundThreshold,
+            downSampleFactor,
+            edgeBlurAmount,
+            effects: videoSegmentation,
+            foregroundThreshold,
+            height: processingHeight,
+            lightWrapBlurAmount,
+            lightWrapEdgeBand,
+            lightWrapIntensity,
+            lightWrapTightness,
             maskCombineRatio,
+            morphDilateRadiusPx,
+            morphErodeRadiusPx,
+            morphPass,
+            personCenter: personCenterX !== undefined && personCenterY !== undefined
+                ? [personCenterX, personCenterY]
+                : undefined,
+            sigmaRangeHi,
+            sigmaRangeLo,
+            sigmaSpace,
+            width: processingWidth,
         });
     const proxy = proxyWithLog(logger, label);
-    let _videoProcessor = undefined;
+    let _videoProcessor;
     const props = {
         videoSegmentationModel,
         segmenters: {
@@ -186,11 +279,25 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
         },
         videoSegmentation,
         backgroundBlurAmount,
+        backgroundImageUrl,
+        backgroundThreshold,
+        downSampleFactor,
         edgeBlurAmount,
         foregroundThreshold,
         frameRate,
-        backgroundImageUrl,
+        lightWrapBlurAmount,
+        lightWrapEdgeBand,
+        lightWrapIntensity,
+        lightWrapTightness,
         maskCombineRatio,
+        morphDilateRadiusPx,
+        morphErodeRadiusPx,
+        morphPass,
+        personCenterX,
+        personCenterY,
+        sigmaRangeHi,
+        sigmaRangeLo,
+        sigmaSpace,
         hasInitialized: options.hasInitializedDeps ?? false,
     };
     return async (prevMediaTrack) => {
@@ -227,7 +334,16 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
                         });
                         if (props.backgroundBlurAmount !== undefined) {
                             props.transformer.backgroundBlurAmount =
-                                getBlurKernelSize(props.backgroundBlurAmount, height, lowestProcessingHeight);
+                                getBlurKernelSize(props.backgroundBlurAmount, referenceProcessingHeight ??
+                                    processingHeight, height);
+                        }
+                        if (props.lightWrapBlurAmount !== undefined) {
+                            props.transformer.lightWrapBlurAmount =
+                                getBlurKernelSize(props.lightWrapBlurAmount, referenceProcessingHeight ??
+                                    processingHeight, height);
+                        }
+                        if (props.edgeBlurAmount !== undefined) {
+                            props.transformer.edgeBlurAmount = getTentBlurSize(props.edgeBlurAmount, referenceProcessingHeight ?? processingHeight, height);
                         }
                     }
                 }
@@ -238,7 +354,7 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
             }
             props.hasInitialized = true;
         }
-        applyFeatures(props.transformer, features, lowestProcessingHeight);
+        applyFeatures(props.transformer, features, referenceProcessingHeight);
         const model = features.videoSegmentationModel &&
             props.segmenters[features.videoSegmentationModel];
         if (features.videoSegmentationModel &&
@@ -318,7 +434,7 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
                 if (isEmpty(features)) {
                     return;
                 }
-                applyFeatures(props.transformer, features, lowestProcessingHeight);
+                applyFeatures(props.transformer, features, referenceProcessingHeight);
                 if (dynamicProcessingDimensions()) {
                     const acceptedSettings = this.previousMediaTrack?.source.getSettings();
                     if (acceptedSettings?.width && acceptedSettings?.height) {
@@ -328,7 +444,16 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
                         });
                         if (props.backgroundBlurAmount !== undefined) {
                             props.transformer.backgroundBlurAmount =
-                                getBlurKernelSize(props.backgroundBlurAmount, acceptedSettings.height, lowestProcessingHeight);
+                                getBlurKernelSize(props.backgroundBlurAmount, referenceProcessingHeight ??
+                                    processingHeight, acceptedSettings.height);
+                        }
+                        if (props.lightWrapBlurAmount !== undefined) {
+                            props.transformer.lightWrapBlurAmount =
+                                getBlurKernelSize(props.lightWrapBlurAmount, referenceProcessingHeight ??
+                                    processingHeight, acceptedSettings.height);
+                        }
+                        if (props.edgeBlurAmount !== undefined) {
+                            props.transformer.edgeBlurAmount = getTentBlurSize(props.edgeBlurAmount, referenceProcessingHeight ?? processingHeight, acceptedSettings.height);
                         }
                     }
                 }
@@ -343,18 +468,35 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
             getSettings: () => {
                 const contentHint = track.contentHint ?? '';
                 return {
-                    videoSegmentation: transformer.effects,
-                    foregroundThreshold: transformer.foregroundThreshold,
-                    // Background blur amount is a function of height
                     backgroundBlurAmount: props.backgroundBlurAmount,
+                    backgroundImageUrl: transformer.backgroundImageUrl,
+                    backgroundThreshold: transformer.backgroundThreshold,
+                    contentHint,
+                    downSampleFactor: transformer.downSampleFactor,
                     edgeBlurAmount: transformer.edgeBlurAmount,
+                    excludeBystanders: transformer.excludeBystanders,
+                    foregroundThreshold: transformer.foregroundThreshold,
+                    frameRate: props.frameRate,
+                    height: props.height,
+                    lightWrapBlurAmount: transformer.lightWrapBlurAmount,
+                    lightWrapEdgeBand: transformer.lightWrapEdgeBand,
+                    lightWrapIntensity: transformer.lightWrapIntensity,
+                    lightWrapTightness: transformer.lightWrapTightness,
                     maskCombineRatio: transformer.maskCombineRatio,
-                    backgroundImageUrl: transformer.backgroundImageUrl,
-                    videoSegmentationModel: transformer.segmenter.modelAsset.modelName,
+                    morphDilateRadiusPx: transformer.morphDilateRadiusPx,
+                    morphErodeRadiusPx: transformer.morphErodeRadiusPx,
+                    morphPass: transformer.morphPass,
                     pan: props.pan,
+                    personCenterX: transformer.personCenter[0],
+                    personCenterY: transformer.personCenter[1],
+                    sigmaRangeHi: transformer.sigmaRangeHi,
+                    sigmaRangeLo: transformer.sigmaRangeLo,
+                    sigmaSpace: transformer.sigmaSpace,
                     tilt: props.tilt,
+                    videoSegmentation: transformer.effects,
+                    videoSegmentationModel: transformer.segmenter.modelAsset.modelName,
+                    width: props.width,
                     zoom: props.zoom,
-                    contentHint,
                 };
             },
         });

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@pexip/media",
-  "version": "20.3.4",
+  "version": "21.0.0",
   "description": "Home for media related stuff",
-  "homepage": "https://gitlab.com/pexip/zoo",
+  "homepage": "https://developer.pexip.com",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
   "repository": {
     "type": "git",
@@ -26,37 +26,42 @@
   },
   "directories": {
     "lib": "lib",
-    "test": "__tests__"
+    "test": "__tests__",
+    "doc": "api-docs"
   },
   "files": [
-    "dist"
+    "dist",
+    "api-docs"
   ],
   "scripts": {
     "build": "pexip-bundler build",
     "build:sourcemap": "pexip-bundler build --sourcemap",
-    "clean": "rm -fr dist",
+    "clean": "rm -fr dist api-docs",
     "dev": "pexip-bundler dev",
-    "prepack": "yarn clean && yarn build",
+    "prepack": "yarn clean && yarn build && yarn docs",
     "test": "jest",
     "check-format": "prettier --ignore-path=../../.prettierignore --check .",
     "format": "prettier --ignore-path=../../.prettierignore --write .",
     "lint": "yarn run -T biome lint",
-    "typecheck": "yarn tsc --noEmit -p ."
+    "typecheck": "yarn tsc --noEmit -p .",
+    "docs": "typedoc"
   },
   "dependencies": {
-    "@pexip/media-control": "20.3.4",
-    "@pexip/media-processor": "20.3.4",
-    "@pexip/signal": "16.9.2",
-    "@pexip/utils": "17.1.2"
+    "@pexip/media-control": "21.0.0",
+    "@pexip/media-processor": "21.0.0",
+    "@pexip/signal": "16.9.4",
+    "@pexip/utils": "17.2.0"
   },
   "devDependencies": {
     "@jest/globals": "^30.2.0",
-    "@pexip/bundler": "18.1.2",
+    "@pexip/bundler": "18.2.0",
     "@swc/core": "^1.10.12",
     "@swc/jest": "^0.2.39",
     "jest": "^29.5.12",
     "jest-junit": "^16.0.0",
     "prettier": "^3.2.5",
+    "typedoc": "^0.28.15",
+    "typedoc-plugin-markdown": "^4.9.0",
     "typescript": "~5.7.3"
   },
   "publishConfig": {