@pexip/media 18.5.0 → 19.1.0

package/dist/videoProcessor.d.ts

@@ -1,6 +1,6 @@
 import type { VideoProcessor, SegmentationTransform, SegmentationModel } from '@pexip/media-processor';
 import type { MediaDeviceRequest } from '@pexip/media-control';
-import type { Process, Media, VideoRenderParams, Segmenters, VideoStreamTrackProcessorAPIs, VideoContentHint } from './types';
+import type { TrackProcessor, VideoRenderParams, Segmenters, VideoStreamTrackProcessorAPIs, VideoContentHint } from './types';
 interface ProcessorDeps {
     videoProcessor?: () => VideoProcessor;
     transformer?: SegmentationTransform;
@@ -18,16 +18,12 @@ interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<Pro
      * Whether or to enable this processor
      */
     shouldEnable: () => boolean;
-    /**
-     * Callback when error occurs
-     */
-    onError?: (error: Error) => void;
     processingWidth: number;
     processingHeight: number;
     hasInitializedDeps?: boolean;
     width?: number;
     height?: number;
-    scope?: string;
+    label?: string;
 }
 interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<ProcessorDeps> {
     hasInitialized: boolean;
@@ -37,5 +33,5 @@ declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUr
 type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
 type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest['video'], props: FeatureProps) => FeatureProps;
-export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope, ...options }: VideoStreamProcessOptions) => Process<Promise<Media>>;
+export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label, ...options }: VideoStreamProcessOptions) => TrackProcessor;
 export {};

package/dist/videoProcessor.js

@@ -1,9 +1,10 @@
 import { createVideoProcessor, createCanvasTransform, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
 import { muteStreamTrack, extractConstraintsWithKeys, getValueFromConstrainNumber, } from '@pexip/media-control';
-import { isEmpty } from '@pexip/utils';
-import { shallowCopy, wrapToJSON, applyExtendedConstraints, getBlurKernelSize, } from './utils';
+import { isEmpty, assert } from '@pexip/utils';
+import { createMediaTrack, getBlurKernelSize } from './utils';
 import { logger, proxyWithLog } from './logger';
 import { isVideoContentHint } from './typeGuard';
+import { PROCESSOR_LABELS } from './constants';
 const FEATURE_KEYS = [
     'backgroundBlurAmount',
     'backgroundImageUrl',
@@ -26,7 +27,7 @@ export const updateFeatureProps = (constraints, props) => {
     return FEATURE_KEYS.reduce((accm, key) => {
         switch (key) {
             case 'contentHint': {
-                const [[feature] = []] = extracted[key];
+                const [[feature = ''] = []] = extracted[key];
                 if (isVideoContentHint(feature) && props[key] !== feature) {
                     props[key] = feature;
                     return { ...accm, [key]: feature };
@@ -130,34 +131,28 @@ const applyFeatures = (transformer, features) => {
         }
     });
 };
-const adjustResolution = async (media, features, processingSize) => {
+const adjustResolution = async (track, features, processingSize) => {
     if (features.videoSegmentation) {
-        const { video: [videoSettings], } = media.getSettings();
-        const constraints = updateFeatureProps(media.constraints?.video, {});
+        const videoSettings = track.getSettings();
+        const constraints = updateFeatureProps(track.getConstraints(), {});
         switch (features.videoSegmentation) {
             case 'blur':
             case 'overlay': {
                 if (videoSettings?.height !== processingSize.height) {
                     try {
-                        await media.applyConstraints({
-                            video: {
-                                width: processingSize.width,
-                                height: processingSize.height,
-                            },
+                        await track.applyConstraints({
+                            width: processingSize.width,
+                            height: processingSize.height,
                         });
-                        const { video: [postVideoSettings], } = media.getSettings();
+                        const postVideoSettings = track.getSettings();
                         if (postVideoSettings?.height !== processingSize.height) {
                             // Workaround Firefox 16:9 ratio https://bugzilla.mozilla.org/show_bug.cgi?id=1193640
-                            await media.applyConstraints({
-                                video: { height: 720 },
-                            });
+                            await track.applyConstraints({ height: 720 });
                         }
                     }
                     catch (error) {
                         // Workaround Firefox 16:9 ratio https://bugzilla.mozilla.org/show_bug.cgi?id=1193640
-                        await media.applyConstraints({
-                            video: { height: 720 },
-                        });
+                        await track.applyConstraints({ height: 720 });
                     }
                 }
                 break;
@@ -165,11 +160,7 @@ const adjustResolution = async (media, features, processingSize) => {
             case 'none': {
                 if (constraints.height &&
                     constraints.height !== videoSettings?.height) {
-                    await media.applyConstraints({
-                        video: {
-                            height: constraints.height,
-                        },
-                    });
+                    await track.applyConstraints({ height: constraints.height });
                 }
                 break;
             }
@@ -187,7 +178,7 @@ export const createVideoStreamProcess = ({ trackProcessorAPI = () => 'stream', p
 //backgroundBlurAmount,
 videoSegmentation, 
 //edgeBlurAmount,
-foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope = 'media', ...options }) => {
+foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label = PROCESSOR_LABELS.VideoProcessor, ...options }) => {
     const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
     const segmenter = options.segmenters[videoSegmentationModel];
     if (!segmenter) {
@@ -206,7 +197,7 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope
             backgroundImageUrl,
             maskCombineRatio,
         });
-    const proxy = proxyWithLog(logger, scope);
+    const proxy = proxyWithLog(logger, label);
     let _videoProcessor = undefined;
     const props = {
         videoSegmentationModel,
@@ -236,127 +227,100 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope
         maskCombineRatio,
         hasInitialized: options.hasInitializedDeps ?? false,
     };
-    return async (mediaP) => {
-        const media = await mediaP;
-        const features = updateFeatureProps(media.constraints?.video, props);
+    return async (prevMediaTrack) => {
+        const features = updateFeatureProps(prevMediaTrack.getConstraints(), props);
         const shouldEnabled = shouldEnable();
-        if (!shouldEnabled || !media.stream?.getVideoTracks().length) {
-            logger.debug({ scope, features, shouldEnabled }, 'Video processing is skipped');
-            return media;
+        if (!shouldEnabled || !prevMediaTrack.track) {
+            logger.debug({ label, features, shouldEnabled }, 'Video processing is skipped');
+            return prevMediaTrack;
         }
-        try {
-            if (!props.hasInitialized) {
-                await props.videoProcessor().open();
-                props.hasInitialized = true;
-            }
-            applyFeatures(props.transformer, features);
-            await adjustResolution(media, features, {
-                width: processingWidth,
-                height: processingHeight,
-            });
-            const model = features.videoSegmentationModel &&
-                props.segmenters[features.videoSegmentationModel];
-            if (features.videoSegmentationModel &&
-                features.videoSegmentationModel !==
-                    props.transformer.segmenter.modelAsset.modelName &&
-                model) {
-                props.transformer.segmenter = model;
-            }
-            const stream = await props.videoProcessor().process(media.stream);
-            const release = async () => {
-                props.videoProcessor().close();
-                await media.release();
-                _videoProcessor = undefined;
-                props.hasInitialized = false;
-            };
-            const muteAudio = (mute) => {
-                media.muteAudio(mute);
-                muteStreamTrack(stream)(mute, 'audio');
-            };
-            const muteVideo = (mute) => {
-                media.muteVideo(mute);
-                props.transformer.effects = mute
-                    ? 'none'
-                    : props.videoSegmentation ?? 'none';
-                muteStreamTrack(stream)(mute, 'video');
-            };
-            const applyConstraints = applyExtendedConstraints(media, async (constraints) => {
-                if (isEmpty(constraints.video)) {
+        if (!props.hasInitialized) {
+            await props.videoProcessor().open();
+            props.hasInitialized = true;
+        }
+        applyFeatures(props.transformer, features);
+        await adjustResolution(prevMediaTrack, features, {
+            width: processingWidth,
+            height: processingHeight,
+        });
+        const model = features.videoSegmentationModel &&
+            props.segmenters[features.videoSegmentationModel];
+        if (features.videoSegmentationModel &&
+            features.videoSegmentationModel !==
+                props.transformer.segmenter.modelAsset.modelName &&
+            model) {
+            props.transformer.segmenter = model;
+        }
+        const stream = await props
+            .videoProcessor()
+            // FIXME: Change process to accept MediaStreamTrack instead of MediaStream
+            .process(new MediaStream([prevMediaTrack.track]));
+        const track = stream.getVideoTracks().at(0);
+        assert(track, 'Video track should be there');
+        // Inherit contentHint from previous track
+        track.contentHint = prevMediaTrack.track.contentHint;
+        const release = async () => {
+            props.videoProcessor().close();
+            await prevMediaTrack.release();
+            _videoProcessor = undefined;
+            props.hasInitialized = false;
+        };
+        const mute = (mute) => {
+            props.transformer.effects = mute
+                ? 'none'
+                : props.videoSegmentation ?? 'none';
+            prevMediaTrack.mute(mute);
+            muteStreamTrack(stream)(mute, 'video');
+        };
+        return createMediaTrack({
+            label,
+            kind: 'videoinput',
+            constraints: prevMediaTrack.getConstraints(),
+            previousMediaTrack: prevMediaTrack,
+            input: prevMediaTrack.input,
+            expectedInput: prevMediaTrack.expectedInput,
+            track,
+            mute,
+            release,
+            applyConstraints: async (constraints) => {
+                if (isEmpty(constraints)) {
                     return;
                 }
-                const features = updateFeatureProps(constraints.video, props);
-                logger.debug({ scope, constraints: constraints.video, features }, 'apply video constraints');
+                const features = updateFeatureProps(constraints, props);
+                logger.debug({ label, constraints: constraints, features }, 'apply video constraints');
                 if (isEmpty(features)) {
                     return;
                 }
-                try {
-                    applyFeatures(props.transformer, features);
-                    await adjustResolution(media, features, {
-                        width: processingWidth,
-                        height: processingHeight,
-                    });
-                    const model = features.videoSegmentationModel &&
-                        props.segmenters[features.videoSegmentationModel];
-                    if (model &&
-                        features.videoSegmentationModel !==
-                            props.transformer.segmenter.modelAsset.modelName) {
-                        props.transformer.segmenter = model;
-                    }
-                }
-                catch (error) {
-                    if (error instanceof Error) {
-                        logger.error({
-                            scope,
-                            constraints: constraints.video,
-                            features,
-                            error,
-                        }, 'failed to apply video constraints');
-                        options.onError?.(error);
-                    }
+                applyFeatures(props.transformer, features);
+                await adjustResolution(prevMediaTrack, features, {
+                    width: processingWidth,
+                    height: processingHeight,
+                });
+                const model = features.videoSegmentationModel &&
+                    props.segmenters[features.videoSegmentationModel];
+                if (model &&
+                    features.videoSegmentationModel !==
+                        props.transformer.segmenter.modelAsset.modelName) {
+                    props.transformer.segmenter = model;
                 }
-            });
-            const prevGetSettings = media.getSettings;
-            return wrapToJSON(shallowCopy(media, {
-                stream,
-                muteAudio,
-                muteVideo,
-                applyConstraints,
-                release,
-                getSettings: () => {
-                    const { audio, video } = prevGetSettings();
-                    const contentHint = stream.getAudioTracks().at(0)
-                        ?.contentHint ?? '';
-                    const videoSettings = {
-                        videoSegmentation: transformer.effects,
-                        foregroundThreshold: transformer.foregroundThreshold,
-                        // Background blur amount is a function of height
-                        backgroundBlurAmount: props.backgroundBlurAmount,
-                        edgeBlurAmount: transformer.edgeBlurAmount,
-                        maskCombineRatio: transformer.maskCombineRatio,
-                        backgroundImageUrl: transformer.backgroundImageUrl,
-                        videoSegmentationModel: transformer.segmenter.modelAsset.modelName,
-                        pan: props.pan,
-                        tilt: props.tilt,
-                        zoom: props.zoom,
-                        contentHint,
-                    };
-                    const settings = {
-                        audio,
-                        video: video.map(settings => ({
-                            ...settings,
-                            ...videoSettings,
-                        })),
-                    };
-                    logger.debug({ scope, settings: videoSettings }, 'get video processor settings');
-                    return settings;
-                },
-            }));
-        }
-        catch (e) {
-            if (e instanceof Error) {
-                options.onError?.(e);
-            }
-            return media;
-        }
+            },
+            getSettings: () => {
+                const contentHint = track.contentHint ?? '';
+                return {
+                    videoSegmentation: transformer.effects,
+                    foregroundThreshold: transformer.foregroundThreshold,
+                    // Background blur amount is a function of height
+                    backgroundBlurAmount: props.backgroundBlurAmount,
+                    edgeBlurAmount: transformer.edgeBlurAmount,
+                    maskCombineRatio: transformer.maskCombineRatio,
+                    backgroundImageUrl: transformer.backgroundImageUrl,
+                    videoSegmentationModel: transformer.segmenter.modelAsset.modelName,
+                    pan: props.pan,
+                    tilt: props.tilt,
+                    zoom: props.zoom,
+                    contentHint,
+                };
+            },
+        });
     };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "18.5.0",
+  "version": "19.1.0",
   "description": "Home for media related stuff",
   "homepage": "https://gitlab.com/pexip/zoo",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -33,13 +33,14 @@
     "typecheck": "yarn tsc --noEmit -p ."
   },
   "dependencies": {
-    "@pexip/media-control": "18.5.0",
-    "@pexip/media-processor": "18.5.0",
+    "@pexip/media-control": "19.1.0",
+    "@pexip/media-processor": "19.1.0",
     "@pexip/signal": "16.7.1",
-    "@pexip/utils": "16.12.1"
+    "@pexip/utils": "16.13.0"
   },
   "devDependencies": {
-    "@pexip/bundler": "17.0.0",
+    "@jest/globals": "^29.7.0",
+    "@pexip/bundler": "17.1.0",
     "@swc/core": "^1.4.6",
     "@swc/jest": "^0.2.36",
     "jest": "^29.5.12",