@pexip/media 19.0.0 → 20.0.0

package/dist/utils.js

@@ -3,22 +3,22 @@ import { calculateMaxBlurPass } from '@pexip/media-processor';
 import { hasOwn, assert, isEmpty } from '@pexip/utils';
 import { internalSignals } from './signals';
 export const makeDeriveDeviceStatus = (constraints) => (audio, video, both) => {
-    if (constraints.audio === undefined || constraints.audio) {
-        if (constraints.video === undefined || constraints.video) {
+    if (constraints.audio) {
+        if (constraints.video) {
             return both;
         }
         return audio;
     }
-    if (constraints.video === undefined || constraints.video) {
+    if (constraints.video) {
         return video;
     }
     return both;
 };
 export const createMediaTrack = (trackInit) => {
-    assert(trackInit.track === undefined ||
+    const trackConsistency = trackInit.track === undefined ||
         (trackInit.kind === trackInit.input?.kind &&
-            trackInit.kind === toMediaDeviceInputKind(trackInit.track),
-            'Inconsistent track kind'));
+            trackInit.kind === toMediaDeviceInputKind(trackInit.track));
+    assert(trackConsistency, `Inconsistent track kind: ${trackInit.input?.kind} ${trackInit.track && toMediaDeviceInputKind(trackInit.track)} vs ${trackInit.kind}`);
     const currentConstraints = typeof trackInit.constraints === 'boolean' ? {} : trackInit.constraints;
     // Find the original source track through the linked list
     let sourceMediaTrack = trackInit.previousMediaTrack;
@@ -41,7 +41,7 @@ export const createMediaTrack = (trackInit) => {
             ...(trackInit.getSettings?.() ?? {}),
         };
     };
-    return {
+    const mediaTrack = {
         get kind() {
             return trackInit.kind;
         },
@@ -52,7 +52,9 @@ export const createMediaTrack = (trackInit) => {
             return trackInit.track;
         },
         get input() {
-            return trackInit.input;
+            return trackInit.track?.readyState !== 'live'
+                ? undefined
+                : trackInit.input;
         },
         get expectedInput() {
             return trackInit.expectedInput;
@@ -74,7 +76,7 @@ export const createMediaTrack = (trackInit) => {
                 isTrackMuted(trackInit.track));
         },
         get source() {
-            return sourceMediaTrack ?? this;
+            return sourceMediaTrack ?? mediaTrack;
         },
         mute(toMute) {
             if (trackInit.overrideMute) {
@@ -89,11 +91,12 @@ export const createMediaTrack = (trackInit) => {
         getConstraints() {
             return getConstraints();
         },
-        clone() {
+        clone(signals) {
             return createMediaTrack({
                 ...trackInit,
                 constraints: getConstraints(),
                 track: trackInit.track?.clone(),
+                signals,
             });
         },
         async applyConstraints(constraints) {
@@ -152,10 +155,14 @@ export const createMediaTrack = (trackInit) => {
             };
         },
     };
+    return mediaTrack;
 };
 export const createTrackProcessingPipeline = (processors) => async (track) => {
     let lastTrack = track;
     for (const process of processors) {
+        if (lastTrack.track?.readyState !== 'live') {
+            return lastTrack;
+        }
         lastTrack = await process(lastTrack);
     }
     return lastTrack;
@@ -187,7 +194,7 @@ export const findExpectedInput = (devices, constraints, input, kind) => {
  * A utility function to check if the provided track is muted. There are
  * 2 factors to be considered: `MediaStreamTrack['muted']` and `MediaStreamTrack['enabled']`.
  *
- * ```
+ * ```md
  * | muted \ enabled | true  | false |
  * |-----------------| ----- | ----- |
  * |     true        | true  | true  |
@@ -334,23 +341,23 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             return props.originalConstraints;
         },
         isAudioInputUnavailable() {
-            return (mediaInit.permission.audio === 'granted' &&
-                props.devices.size('audioinput') === 0);
+            return !props.devices.anyAuthorizedDevice('audioinput');
         },
         isVideoInputUnavailable() {
-            return (mediaInit.permission.video === 'granted' &&
-                props.devices.size('videoinput') === 0);
+            return !props.devices.anyAuthorizedDevice('videoinput');
         },
         setOriginalConstraints(constraints) {
             props.originalConstraints = constraints;
         },
         requestedAudio() {
             return (Boolean(props.audioTrack?.track) ||
-                Boolean(props.originalConstraints.audio));
+                (Boolean(props.originalConstraints.audio) &&
+                    props.devices.size('audioinput') > 0));
         },
         requestedVideo() {
             return (Boolean(props.videoTrack?.track) ||
-                Boolean(props.originalConstraints.video));
+                (Boolean(props.originalConstraints.video) &&
+                    props.devices.size('videoinput') > 0));
         },
         muteAudio(mute) {
             if (!props.audioTrack) {
@@ -377,7 +384,9 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
         async release() {
             unsubscribe?.();
             unsubscribe = undefined;
-            trackSubscriptions.forEach(unsubscribe => unsubscribe());
+            for (const unsubscribe of trackSubscriptions.values()) {
+                unsubscribe();
+            }
             trackSubscriptions.clear();
             await Promise.all([
                 props.audioTrack?.release(),
@@ -388,12 +397,12 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             audio: props.audioTrack?.getSettings(),
             video: props.videoTrack?.getSettings(),
         }),
-        clone() {
+        clone(signals) {
             const tracks = [props.audioTrack, props.videoTrack].flatMap(track => {
                 if (!track?.source) {
                     return [];
                 }
-                const cloned = track.source.clone();
+                const cloned = track.source.clone(signals);
                 // Restore the enabled state for all cloned track
                 cloned.mute(false);
                 return cloned;
@@ -516,6 +525,11 @@ export const VIDEO_SETTINGS_KEYS = [
 export const MIXING_SETTINGS_KEYS = [
     'mixWithAdditionalMedia',
 ];
+export const PAN_TILT_ZOOM_SETTINGS_KEYS = [
+    'pan',
+    'tilt',
+    'zoom',
+];
 export const hasSettingsChanged = (keysToLookFor) => {
     const cache = {};
     return (settingsA, settingsB) => {
@@ -544,6 +558,28 @@ export const hasSettingsChanged = (keysToLookFor) => {
         return cache.result;
     };
 };
+export const isApplyingRenderEffect = (effect) => {
+    switch (effect) {
+        case 'blur':
+        case 'overlay':
+            return true;
+        default:
+            return false;
+    }
+};
+export const getSettingsFromKeys = (keys, settings) => {
+    if (!settings) {
+        return settings;
+    }
+    const result = {};
+    for (const key of keys) {
+        if (settings[key] !== undefined) {
+            // @ts-expect-error --- Type issue and need to be fixed when we have time
+            result[key] = settings[key];
+        }
+    }
+    return result;
+};
 export const diffSettings = (keysToLookFor) => {
     const cache = {};
     return (settingsA, settingsB) => {
@@ -559,7 +595,16 @@ export const diffSettings = (keysToLookFor) => {
             return cache.diff;
         }
         if (settingsA === undefined || settingsB === undefined) {
-            cache.diff = settingsB;
+            const settings = settingsA ?? settingsB;
+            for (const key of keysToLookFor) {
+                if (settings?.[key] !== undefined) {
+                    if (cache.diff === undefined) {
+                        cache.diff = {};
+                    }
+                    // @ts-expect-error --- Type issue and need to be fixed when we have time
+                    cache.diff[key] = settings[key];
+                }
+            }
             return cache.diff;
         }
         for (const key of keysToLookFor) {
@@ -624,6 +669,7 @@ export const refineMediaConstraints = (state) => {
     switch (state.permission) {
         case 'denied':
             return false;
+        case 'prompt':
         case 'granted': {
             const requestNeeded = shouldRequestDevice({
                 kind: state.kind,
@@ -641,7 +687,6 @@ export const refineMediaConstraints = (state) => {
             }
             return state.request;
         }
-        case 'prompt':
         default:
             return state.request;
     }
@@ -649,25 +694,25 @@ export const refineMediaConstraints = (state) => {
 export const createMediaProcessor = ({ audioProcessors, videoProcessors, onProcessingError, }) => {
     const processAudioTrack = createTrackProcessingPipeline(audioProcessors);
     const processVideoTrack = createTrackProcessingPipeline(videoProcessors);
-    return async (newTracks) => {
-        const processTrack = async (track) => {
-            try {
-                switch (track.kind) {
-                    case 'audioinput':
-                        return await processAudioTrack(track);
-                    case 'videoinput':
-                        return await processVideoTrack(track);
-                    default:
-                        assert(false, `Unexpected track kind: ${track.kind}`);
-                }
+    const processTrack = async (track) => {
+        try {
+            switch (track.kind) {
+                case 'audioinput':
+                    return await processAudioTrack(track);
+                case 'videoinput':
+                    return await processVideoTrack(track);
+                default:
+                    assert(false, `Unexpected track kind: ${track.kind}`);
             }
-            catch (error) {
-                if (error instanceof Error) {
-                    onProcessingError?.(error, track);
-                }
-                return track;
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                onProcessingError?.(error, track);
             }
-        };
+            return track;
+        }
+    };
+    return async (newTracks) => {
         return await Promise.all(newTracks.map(track => processTrack(track)));
     };
 };

package/dist/videoProcessor.d.ts

@@ -1,4 +1,4 @@
-import type { VideoProcessor, SegmentationTransform, SegmentationModel } from '@pexip/media-processor';
+import type { VideoProcessor, SegmentationTransform, SegmentationModel, RenderBackend } from '@pexip/media-processor';
 import type { MediaDeviceRequest } from '@pexip/media-control';
 import type { TrackProcessor, VideoRenderParams, Segmenters, VideoStreamTrackProcessorAPIs, VideoContentHint } from './types';
 interface ProcessorDeps {
@@ -24,6 +24,8 @@ interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<Pro
     width?: number;
     height?: number;
     label?: string;
+    dynamicProcessingDimensions?: () => boolean;
+    gpuAPI?: () => RenderBackend;
 }
 interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<ProcessorDeps> {
     hasInitialized: boolean;
@@ -32,6 +34,6 @@ interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<P
 declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUrl", "maskCombineRatio", "edgeBlurAmount", "foregroundThreshold", "frameRate", "videoSegmentation", "videoSegmentationModel", "width", "height", "pan", "tilt", "zoom", "contentHint"];
 type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
 type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
-export declare const updateFeatureProps: (constraints: MediaDeviceRequest['video'], props: FeatureProps) => FeatureProps;
-export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label, ...options }: VideoStreamProcessOptions) => TrackProcessor;
+export declare const updateFeatureProps: (constraints: MediaDeviceRequest["video"], props: FeatureProps) => FeatureProps;
+export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label, dynamicProcessingDimensions, gpuAPI, ...options }: VideoStreamProcessOptions) => TrackProcessor;
 export {};

package/dist/videoProcessor.js

@@ -30,7 +30,8 @@ export const updateFeatureProps = (constraints, props) => {
                 const [[feature = ''] = []] = extracted[key];
                 if (isVideoContentHint(feature) && props[key] !== feature) {
                     props[key] = feature;
-                    return { ...accm, [key]: feature };
+                    accm[key] = feature;
+                    return accm;
                 }
                 return accm;
             }
@@ -38,7 +39,8 @@ export const updateFeatureProps = (constraints, props) => {
                 const [[feature] = []] = extracted[key];
                 if (isRenderEffects(feature) && props[key] !== feature) {
                     props[key] = feature;
-                    return { ...accm, [key]: feature };
+                    accm[key] = feature;
+                    return accm;
                 }
                 return accm;
             }
@@ -46,7 +48,8 @@ export const updateFeatureProps = (constraints, props) => {
                 const [[feature] = []] = extracted[key];
                 if (isSegmentationModel(feature) && props[key] !== feature) {
                     props[key] = feature;
-                    return { ...accm, [key]: feature };
+                    accm[key] = feature;
+                    return accm;
                 }
                 return accm;
             }
@@ -54,7 +57,8 @@ export const updateFeatureProps = (constraints, props) => {
                 const [[feature] = []] = extracted[key];
                 if (feature) {
                     props[key] = feature;
-                    return { ...accm, [key]: feature };
+                    accm[key] = feature;
+                    return accm;
                 }
                 return accm;
             }
@@ -70,10 +74,8 @@ export const updateFeatureProps = (constraints, props) => {
                     const value = getValueFromConstrainNumber(feature);
                     if (props[key] !== value) {
                         props[key] = value;
-                        return {
-                            ...accm,
-                            [key]: value,
-                        };
+                        accm[key] = value;
+                        return accm;
                     }
                 }
                 return accm;
@@ -84,7 +86,8 @@ export const updateFeatureProps = (constraints, props) => {
                 const [feature] = extracted[key];
                 if (feature !== undefined && props[key] !== feature) {
                     props[key] = feature;
-                    return { ...accm, [key]: feature };
+                    accm[key] = feature;
+                    return accm;
                 }
                 return accm;
             }
@@ -92,7 +95,7 @@ export const updateFeatureProps = (constraints, props) => {
     }, {});
 };
 const applyFeatures = (transformer, features) => {
-    Object.keys(features).forEach(key => {
+    for (const key in features) {
         const k = key;
         switch (k) {
             case 'edgeBlurAmount':
@@ -102,66 +105,30 @@ const applyFeatures = (transformer, features) => {
                 if (value !== undefined) {
                     transformer[k] = value;
                 }
-                return;
+                break;
             }
             case 'backgroundBlurAmount': {
                 const value = features[k];
                 if (value !== undefined) {
                     transformer[k] = getBlurKernelSize(value, transformer.height);
                 }
-                return;
+                break;
             }
             case 'videoSegmentation': {
                 const value = features[k];
                 if (value && value !== transformer.effects) {
                     transformer.effects = value;
                 }
-                return;
+                break;
             }
             case 'backgroundImageUrl': {
                 const value = features[k];
                 if (value && value !== transformer.backgroundImageUrl) {
                     transformer.backgroundImageUrl = value;
                 }
-                return;
-            }
-            default: {
-                return;
-            }
-        }
-    });
-};
-const adjustResolution = async (track, features, processingSize) => {
-    if (features.videoSegmentation) {
-        const videoSettings = track.getSettings();
-        const constraints = updateFeatureProps(track.getConstraints(), {});
-        switch (features.videoSegmentation) {
-            case 'blur':
-            case 'overlay': {
-                if (videoSettings?.height !== processingSize.height) {
-                    try {
-                        await track.applyConstraints({
-                            width: processingSize.width,
-                            height: processingSize.height,
-                        });
-                        const postVideoSettings = track.getSettings();
-                        if (postVideoSettings?.height !== processingSize.height) {
-                            // Workaround Firefox 16:9 ratio https://bugzilla.mozilla.org/show_bug.cgi?id=1193640
-                            await track.applyConstraints({ height: 720 });
-                        }
-                    }
-                    catch (error) {
-                        // Workaround Firefox 16:9 ratio https://bugzilla.mozilla.org/show_bug.cgi?id=1193640
-                        await track.applyConstraints({ height: 720 });
-                    }
-                }
                 break;
             }
-            case 'none': {
-                if (constraints.height &&
-                    constraints.height !== videoSettings?.height) {
-                    await track.applyConstraints({ height: constraints.height });
-                }
+            default: {
                 break;
             }
         }
@@ -178,7 +145,8 @@ export const createVideoStreamProcess = ({ trackProcessorAPI = () => 'stream', p
 //backgroundBlurAmount,
 videoSegmentation, 
 //edgeBlurAmount,
-foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label = PROCESSOR_LABELS.VideoProcessor, ...options }) => {
+foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label = PROCESSOR_LABELS.VideoProcessor, dynamicProcessingDimensions = () => false, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
+...options }) => {
     const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
     const segmenter = options.segmenters[videoSegmentationModel];
     if (!segmenter) {
@@ -235,14 +203,34 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label
             return prevMediaTrack;
         }
         if (!props.hasInitialized) {
-            await props.videoProcessor().open();
+            if (dynamicProcessingDimensions()) {
+                const { width, height } = prevMediaTrack.getSettings();
+                /**
+                 * when the processor is closed, the segmenter is not killed, but the track is released.
+                 * this means we can't listen on the track settings changes when we change quality
+                 * we need to update instead of open if the segmenter is already running
+                 */
+                if (props.transformer.segmenter.status === 'new') {
+                    await props.videoProcessor().open({
+                        processingWidth: width,
+                        processingHeight: height,
+                    });
+                    props.transformer.backend = gpuAPI();
+                }
+                else {
+                    props.transformer.update({
+                        processingWidth: width,
+                        processingHeight: height,
+                    });
+                }
+            }
+            else {
+                await props.videoProcessor().open();
+                props.transformer.backend = gpuAPI();
+            }
             props.hasInitialized = true;
         }
         applyFeatures(props.transformer, features);
-        await adjustResolution(prevMediaTrack, features, {
-            width: processingWidth,
-            height: processingHeight,
-        });
         const model = features.videoSegmentationModel &&
             props.segmenters[features.videoSegmentationModel];
         if (features.videoSegmentationModel &&
@@ -292,10 +280,15 @@ foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label
                     return;
                 }
                 applyFeatures(props.transformer, features);
-                await adjustResolution(prevMediaTrack, features, {
-                    width: processingWidth,
-                    height: processingHeight,
-                });
+                if (dynamicProcessingDimensions()) {
+                    const acceptedSettings = prevMediaTrack.source.getSettings();
+                    if (acceptedSettings.width && acceptedSettings.height) {
+                        props.transformer.update({
+                            processingHeight: acceptedSettings.height,
+                            processingWidth: acceptedSettings.width,
+                        });
+                    }
+                }
                 const model = features.videoSegmentationModel &&
                     props.segmenters[features.videoSegmentationModel];
                 if (model &&

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "19.0.0",
+  "version": "20.0.0",
   "description": "Home for media related stuff",
   "homepage": "https://gitlab.com/pexip/zoo",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -14,6 +14,16 @@
   "author": "Terrence Lam <terrence@tlam.dev>",
   "module": "dist/index.js",
   "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./*": {
+      "types": "./dist/*.d.ts",
+      "import": "./dist/*.js"
+    }
+  },
   "directories": {
     "lib": "lib",
     "test": "__tests__"
@@ -23,30 +33,31 @@
   ],
   "scripts": {
     "build": "pexip-bundler build",
+    "build:sourcemap": "pexip-bundler build --sourcemap",
     "clean": "rm -fr dist",
     "dev": "pexip-bundler dev",
-    "prepack": "yarn clean && yarn build --no-sourcemap",
+    "prepack": "yarn clean && yarn build",
     "test": "jest",
     "check-format": "prettier --ignore-path=../../.prettierignore --check .",
     "format": "prettier --ignore-path=../../.prettierignore --write .",
-    "lint": "yarn run -T eslint --ext .js,.ts,.tsx --report-unused-disable-directives .",
+    "lint": "yarn run -T biome lint",
     "typecheck": "yarn tsc --noEmit -p ."
   },
   "dependencies": {
-    "@pexip/media-control": "19.0.0",
-    "@pexip/media-processor": "19.0.0",
-    "@pexip/signal": "16.7.1",
-    "@pexip/utils": "16.13.0"
+    "@pexip/media-control": "20.0.0",
+    "@pexip/media-processor": "20.0.0",
+    "@pexip/signal": "16.8.0",
+    "@pexip/utils": "17.0.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@pexip/bundler": "17.1.0",
-    "@swc/core": "^1.4.6",
-    "@swc/jest": "^0.2.36",
+    "@pexip/bundler": "18.0.0",
+    "@swc/core": "^1.10.12",
+    "@swc/jest": "^0.2.37",
     "jest": "^29.5.12",
     "jest-junit": "^16.0.0",
     "prettier": "^3.2.5",
-    "typescript": "~5.3.0"
+    "typescript": "~5.7.3"
   },
   "publishConfig": {
     "access": "public",