@pexip/media 17.4.0 → 18.0.0

package/CHANGELOG.md

@@ -1,5 +1,41 @@
 # @pexip/media
 
+## 18.0.0
+
+### Major Changes
+
+- bfb8058: Use the lastest version of media-processor
+
+### Minor Changes
+
+- 12fac2c: Fis VideoProcessor Leak
+
+  Closes https://gitlab.com/pexip/zoo/-/issues/4649
+
+### Patch Changes
+
+- d246fbf: ```ts await props.media.release(); cleanUpMainSubscription();
+
+  ```
+
+  are done in the `cleanup` function and removed from `applyChanges` and `revertChanges`. Cleanup is removed from those functions and needs to be called explicitly.
+  ```
+
+- Updated dependencies [bfb8058]
+- Updated dependencies [79d02d8]
+- Updated dependencies [0a5a136]
+- Updated dependencies [6b75621]
+- Updated dependencies [218a2c0]
+- Updated dependencies [bfb8058]
+- Updated dependencies [bfb8058]
+- Updated dependencies [9bcfc68]
+- Updated dependencies [1543a66]
+- Updated dependencies [68e8ec1]
+  - @pexip/media-processor@18.0.0
+  - @pexip/media-control@18.0.0
+  - @pexip/utils@16.11.0
+  - @pexip/signal@16.7.0
+
 ## 17.4.0
 
 ### Minor Changes

package/dist/previewController.d.ts

@@ -49,6 +49,7 @@ export interface PreviewStreamController {
     updateVideoInput(input: PreviewInput): void;
     applyChanges(force?: boolean): Promise<void>;
     revertChanges(): Promise<void>;
+    cleanup: () => Promise<void>;
     onMediaChanged(callback: EventCallback<Media>): Unsubscribe;
     onAudioInputChanged(callback: EventCallback<PreviewInput>): Unsubscribe;
     onVideoInputChanged(callback: EventCallback<PreviewInput>): Unsubscribe;

package/dist/previewController.js

@@ -246,8 +246,10 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
             props.updatingPreview = false;
         }
     };
-    const cleanup = () => {
+    const cleanup = async () => {
         logger.debug('Cleanup preview controller');
+        await props.media.release();
+        cleanUpMainSubscription();
         props.audioInput = undefined;
         props.videoInput = undefined;
         onEnded?.();
@@ -261,8 +263,6 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
     const applyChanges = async (force = false) => {
         const { audioInput, videoInput } = props;
         const { audio: [previewAudio], video: [previewVideo], } = props.media.getSettings();
-        await props.media.release();
-        cleanUpMainSubscription();
         if (force || hasChanges()) {
             const audioSettings = extractFeaturesToConstraints(AUDIO_SETTINGS_KEYS, previewAudio);
             const videoSettings = extractFeaturesToConstraints(VIDEO_SETTINGS_KEYS, previewVideo);
@@ -286,11 +286,8 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
                 throw error;
             }
         }
-        cleanup();
     };
     const revertChanges = async () => {
-        await props.media.release();
-        cleanUpMainSubscription();
         const mainMedia = getCurrentMedia();
         const mainStream = mainMedia?.stream;
         if (mainMedia && mainStream) {
@@ -323,7 +320,6 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
                 }
             }
         }
-        cleanup();
     };
     const updateInput = (hasChanged, updater) => (input) => {
         if (hasChanged(input)) {
@@ -371,5 +367,6 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
         onUpdatingMain: toEvenHandler('updatingMain'),
         applyChanges,
         revertChanges,
+        cleanup,
     };
 };

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 import type { InputConstraintSet, MediaDeviceInfoLike, MediaDeviceRequest } from '@pexip/media-control';
-import type { RenderParams, RenderEffects, Segmenter, SegmentationModel } from '@pexip/media-processor';
+import type { RendererOptions, RenderEffects, Segmenter, SegmentationModel } from '@pexip/media-processor';
 import type { Signal } from '@pexip/signal';
 export type Unsubscribe = () => void;
 export interface ExtendedMediaTrackSettings extends MediaTrackSettings {
@@ -9,9 +9,9 @@ export interface ExtendedMediaTrackSettings extends MediaTrackSettings {
     vad?: boolean;
     asd?: boolean;
     backgroundBlurAmount?: number;
-    bgImageUrl?: string;
+    backgroundImageUrl?: string;
     edgeBlurAmount?: number;
-    flipHorizontal?: boolean;
+    maskCombineRatio?: number;
     foregroundThreshold?: number;
     videoSegmentation?: RenderEffects;
     videoSegmentationModel?: SegmentationModel;
@@ -417,18 +417,16 @@ export interface MediaController {
      */
     tryAndGetUserMedia: (constraints: MediaDeviceRequest) => void;
 }
-export type VideoRenderParams = Omit<RenderParams, 'backgroundImage' | 'effects'> & {
+export type VideoRenderParams = Omit<RendererOptions, 'effects'> & {
     /**
      * Target frame rate for the video segmentation
      */
     frameRate: number;
-    /**
-     * Default background image URL for overlay effects
-     */
-    bgImageUrl: string;
     /**
      * Render Effects
      */
+    width: number;
+    height: number;
     videoSegmentation: RenderEffects;
     pan?: boolean;
     tilt?: boolean;
@@ -473,9 +471,9 @@ export declare enum DeniedDevices {
     Camera = "camera",
     Both = "microphone-and-camera"
 }
-export interface Segmenters {
-    mediapipeSelfie: Segmenter;
-}
+export type Segmenters = {
+    [Property in SegmentationModel]: Segmenter;
+};
 /**
  * Audio content hints are only applicable when the MediaStreamTrack contains an
  * audio track

package/dist/utils.d.ts

@@ -93,7 +93,7 @@ export declare const wrapToJSON: (media: Media) => Media;
  * @param percentage - The percentage of image height to calculate the blur
  * kernel size
  * @param height - The image height
- * @param max - The max kernel size
+ * @param max - The upper bound
  *
  * @returns blur kernel size
  */

package/dist/utils.js

@@ -1,4 +1,5 @@
 import { applyConstraints, createTrackDevicesChanges, extractConstraintsWithKeys, findDeviceFromConstraints, muteStreamTrack, relaxInputConstraint, stopMediaStream, isAudioInput, isVideoInput, } from '@pexip/media-control';
+import { calculateMaxBlurPass } from '@pexip/media-processor';
 import { UserMediaStatus } from './types';
 import { isOverConstrained } from './status';
 import { isMedia } from './typeGuard';
@@ -275,8 +276,8 @@ export const VIDEO_SETTINGS_KEYS = [
     'foregroundThreshold',
     'backgroundBlurAmount',
     'edgeBlurAmount',
-    'flipHorizontal',
-    'bgImageUrl',
+    'maskCombineRatio',
+    'backgroundImageUrl',
     'width',
     'height',
     'contentHint',
@@ -337,15 +338,15 @@ export const wrapToJSON = (media) => {
  * @param percentage - The percentage of image height to calculate the blur
  * kernel size
  * @param height - The image height
- * @param max - The max kernel size
+ * @param max - The upper bound
  *
  * @returns blur kernel size
  */
-export const getBlurKernelSize = (percentage, height, max = 20) => {
+export const getBlurKernelSize = (percentage, height, max = calculateMaxBlurPass(height)) => {
     if (height <= 0 || percentage <= 0 || max <= 0) {
         return 0;
     }
-    return Math.min(Math.ceil(percentage * 0.001 * height), max);
+    return Math.min(Math.ceil(percentage * 0.01 * max), max);
 };
 /**
  * Apply the content hint to the track

package/dist/videoProcessor.d.ts

@@ -1,10 +1,10 @@
-import type { VideoProcessor, SegmentationModel, SegmentationTransform } from '@pexip/media-processor';
+import type { VideoProcessor, SegmentationTransform, SegmentationModel } from '@pexip/media-processor';
 import type { MediaDeviceRequest } from '@pexip/media-control';
 import type { Process, Media, VideoRenderParams, Segmenters, VideoStreamTrackProcessorAPIs, VideoContentHint } from './types';
 interface ProcessorDeps {
     videoProcessor?: () => VideoProcessor;
     transformer?: SegmentationTransform;
-    segmenters: Segmenters;
+    segmenters: Partial<Segmenters>;
     videoSegmentationModel?: SegmentationModel;
 }
 interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<ProcessorDeps, 'videoProcessor'> {
@@ -33,24 +33,9 @@ interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<P
     hasInitialized: boolean;
     contentHint?: VideoContentHint;
 }
-declare const FEATURE_KEYS: [
-    'backgroundBlurAmount',
-    'bgImageUrl',
-    'edgeBlurAmount',
-    'flipHorizontal',
-    'foregroundThreshold',
-    'frameRate',
-    'videoSegmentation',
-    'videoSegmentationModel',
-    'width',
-    'height',
-    'pan',
-    'tilt',
-    'zoom',
-    'contentHint'
-];
+declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUrl", "maskCombineRatio", "edgeBlurAmount", "foregroundThreshold", "frameRate", "videoSegmentation", "videoSegmentationModel", "width", "height", "pan", "tilt", "zoom", "contentHint"];
 type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
 type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest['video'], props: FeatureProps) => FeatureProps;
-export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, bgImageUrl, flipHorizontal, edgeBlurAmount, scope, ...options }: VideoStreamProcessOptions) => Process<Promise<Media>>;
+export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope, ...options }: VideoStreamProcessOptions) => Process<Promise<Media>>;
 export {};

package/dist/videoProcessor.js

@@ -6,9 +6,9 @@ import { logger, proxyWithLog } from './logger';
 import { isVideoContentHint } from './typeGuard';
 const FEATURE_KEYS = [
     'backgroundBlurAmount',
-    'bgImageUrl',
+    'backgroundImageUrl',
+    'maskCombineRatio',
     'edgeBlurAmount',
-    'flipHorizontal',
     'foregroundThreshold',
     'frameRate',
     'videoSegmentation',
@@ -49,7 +49,7 @@ export const updateFeatureProps = (constraints, props) => {
                 }
                 return accm;
             }
-            case 'bgImageUrl': {
+            case 'backgroundImageUrl': {
                 const [[feature] = []] = extracted[key];
                 if (feature) {
                     props[key] = feature;
@@ -62,6 +62,7 @@ export const updateFeatureProps = (constraints, props) => {
             case 'frameRate':
             case 'foregroundThreshold':
             case 'edgeBlurAmount':
+            case 'maskCombineRatio':
             case 'backgroundBlurAmount': {
                 const [feature] = extracted[key];
                 if (feature !== undefined) {
@@ -76,7 +77,6 @@ export const updateFeatureProps = (constraints, props) => {
                 }
                 return accm;
             }
-            case 'flipHorizontal':
             case 'pan':
             case 'tilt':
             case 'zoom': {
@@ -90,16 +90,12 @@ export const updateFeatureProps = (constraints, props) => {
         }
     }, {});
 };
-const applyFeatures = async (transformer, features) => {
-    // bgImageUrl should be applied before `videoSegmentation` effects since
-    // backgroundImage is the prerequisite of the `overlay` effects
-    if (features.bgImageUrl) {
-        await transformer.loadBackgroundImage(features.bgImageUrl);
-    }
+const applyFeatures = (transformer, features) => {
     Object.keys(features).forEach(key => {
         const k = key;
         switch (k) {
             case 'edgeBlurAmount':
+            case 'maskCombineRatio':
             case 'foregroundThreshold': {
                 const value = features[k];
                 if (value !== undefined) {
@@ -121,11 +117,10 @@ const applyFeatures = async (transformer, features) => {
                 }
                 return;
             }
-            case 'flipHorizontal': {
+            case 'backgroundImageUrl': {
                 const value = features[k];
-                if (value !== undefined &&
-                    value !== transformer.flipHorizontal) {
-                    transformer.flipHorizontal = value;
+                if (value && value !== transformer.backgroundImageUrl) {
+                    transformer.backgroundImageUrl = value;
                 }
                 return;
             }
@@ -192,39 +187,53 @@ export const createVideoStreamProcess = ({ trackProcessorAPI = () => 'stream', p
 //backgroundBlurAmount,
 videoSegmentation, 
 //edgeBlurAmount,
-foregroundThreshold, bgImageUrl, flipHorizontal, edgeBlurAmount, scope = 'media', ...options }) => {
-    const videoSegmentationModel = options.videoSegmentationModel ?? 'mediapipeSelfie';
+foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope = 'media', ...options }) => {
+    const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
+    const segmenter = options.segmenters[videoSegmentationModel];
+    if (!segmenter) {
+        throw new Error('Segmenter is undefined');
+    }
     const backgroundBlurAmount = options.backgroundBlurAmount &&
         getBlurKernelSize(options.backgroundBlurAmount, processingHeight);
     const transformer = options.transformer ??
-        createCanvasTransform(options.segmenters[videoSegmentationModel], {
+        createCanvasTransform(segmenter, {
             width: processingWidth,
             height: processingHeight,
             effects: videoSegmentation,
             foregroundThreshold,
             backgroundBlurAmount,
             edgeBlurAmount,
-            flipHorizontal,
+            backgroundImageUrl,
+            maskCombineRatio,
         });
     const proxy = proxyWithLog(logger, scope);
+    let _videoProcessor = undefined;
     const props = {
         videoSegmentationModel,
         segmenters: {
-            mediapipeSelfie: proxy(options.segmenters.mediapipeSelfie, 'Segmenter'),
+            selfie: options.segmenters.selfie &&
+                proxy(options.segmenters.selfie, 'Segmenter'),
+            deeplabV3: options.segmenters.deeplabV3 &&
+                proxy(options.segmenters.deeplabV3, 'Segmenter'),
         },
         transformer: proxy(transformer, 'Transformer'),
-        videoProcessor: () => proxy(createVideoProcessor([transformer], getTrackProcessor(trackProcessorAPI() === 'stream', {
-            width: processingWidth,
-            height: processingHeight,
-            frameRate,
-        })), 'VideoProcessor'),
+        videoProcessor: () => {
+            if (!_videoProcessor) {
+                _videoProcessor = proxy(createVideoProcessor([transformer], getTrackProcessor(trackProcessorAPI() === 'stream', {
+                    width: processingWidth,
+                    height: processingHeight,
+                    frameRate,
+                })), 'VideoProcessor');
+            }
+            return _videoProcessor;
+        },
         videoSegmentation,
         backgroundBlurAmount,
         edgeBlurAmount,
         foregroundThreshold,
         frameRate,
-        bgImageUrl,
-        flipHorizontal,
+        backgroundImageUrl,
+        maskCombineRatio,
         hasInitialized: options.hasInitializedDeps ?? false,
     };
     return async (mediaP) => {
@@ -240,21 +249,24 @@ foregroundThreshold, bgImageUrl, flipHorizontal, edgeBlurAmount, scope = 'media'
                 await props.videoProcessor().open();
                 props.hasInitialized = true;
             }
-            await applyFeatures(props.transformer, features);
+            applyFeatures(props.transformer, features);
             await adjustResolution(media, features, {
                 width: processingWidth,
                 height: processingHeight,
             });
+            const model = features.videoSegmentationModel &&
+                props.segmenters[features.videoSegmentationModel];
             if (features.videoSegmentationModel &&
                 features.videoSegmentationModel !==
-                    props.transformer.segmenter.model) {
-                props.transformer.segmenter =
-                    props.segmenters[features.videoSegmentationModel];
+                    props.transformer.segmenter.modelAsset.modelName &&
+                model) {
+                props.transformer.segmenter = model;
             }
             const stream = await props.videoProcessor().process(media.stream);
             const release = async () => {
                 props.videoProcessor().close();
                 await media.release();
+                _videoProcessor = undefined;
                 props.hasInitialized = false;
             };
             const muteAudio = (mute) => {
@@ -278,16 +290,17 @@ foregroundThreshold, bgImageUrl, flipHorizontal, edgeBlurAmount, scope = 'media'
                     return;
                 }
                 try {
-                    await applyFeatures(props.transformer, features);
+                    applyFeatures(props.transformer, features);
                     await adjustResolution(media, features, {
                         width: processingWidth,
                         height: processingHeight,
                     });
-                    if (features.videoSegmentationModel &&
+                    const model = features.videoSegmentationModel &&
+                        props.segmenters[features.videoSegmentationModel];
+                    if (model &&
                         features.videoSegmentationModel !==
-                            props.transformer.segmenter.model) {
-                        props.transformer.segmenter =
-                            props.segmenters[features.videoSegmentationModel];
+                            props.transformer.segmenter.modelAsset.modelName) {
+                        props.transformer.segmenter = model;
                     }
                 }
                 catch (error) {
@@ -319,9 +332,9 @@ foregroundThreshold, bgImageUrl, flipHorizontal, edgeBlurAmount, scope = 'media'
                         // Background blur amount is a function of height
                         backgroundBlurAmount: props.backgroundBlurAmount,
                         edgeBlurAmount: transformer.edgeBlurAmount,
-                        flipHorizontal: transformer.flipHorizontal,
-                        bgImageUrl: transformer.backgroundImage && props.bgImageUrl,
-                        videoSegmentationModel: transformer.segmenter.model,
+                        maskCombineRatio: transformer.maskCombineRatio,
+                        backgroundImageUrl: transformer.backgroundImageUrl,
+                        videoSegmentationModel: transformer.segmenter.modelAsset.modelName,
                         pan: props.pan,
                         tilt: props.tilt,
                         zoom: props.zoom,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "17.4.0",
+  "version": "18.0.0",
   "description": "Home for media related stuff",
   "homepage": "https://gitlab.com/pexip/zoo",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -32,14 +32,14 @@
     "typecheck": "yarn tsc --noEmit -p ."
   },
   "dependencies": {
-    "@pexip/media-control": "17.4.0",
-    "@pexip/media-processor": "17.4.0",
+    "@pexip/media-control": "18.0.0",
+    "@pexip/media-processor": "18.0.0",
     "@pexip/signal": "16.7.0",
-    "@pexip/utils": "16.10.0"
+    "@pexip/utils": "16.11.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@pexip/bundler": "16.6.0",
+    "@pexip/bundler": "16.6.1",
     "prettier": "^3.2.5",
     "typescript": "~5.3.0"
   },