@pexip/media 20.1.0 → 20.3.0

package/dist/utils.d.ts

@@ -28,6 +28,7 @@ export declare const findExpectedInput: (devices: IndexedDevices, constraints: I
  * there is no track to check
  */
 export declare const isTrackMuted: (track: MediaStreamTrack | undefined) => boolean | undefined;
+export declare const isTrackEnded: (track: MediaStreamTrack) => boolean;
 export declare const buildMedia: (mediaInit: MediaInit, onDevicesChanged?: import("@pexip/signal").Signal<IndexedDevices, IndexedDevices>) => Media;
 /**
  * Shallow copy the provided object and override with provided overriding
@@ -58,11 +59,12 @@ export declare const mergeSettings: (settingsA: ExtendedMediaTrackSettings | und
  * @param percentage - The percentage of image height to calculate the blur
  * kernel size
  * @param height - The image height
+ * @param lowestProcessingHeight - The lowest image height that is processed by the consumer, used to adjust the blur kernel size across different resolutions
  * @param max - The upper bound
  *
  * @returns blur kernel size
  */
-export declare const getBlurKernelSize: (percentage: number, height: number, max?: number) => number;
+export declare const getBlurKernelSize: (percentage: number, height: number, lowestProcessingHeight?: number, max?: number) => number;
 /**
  * Apply the content hint to the track
  *

package/dist/utils.js

@@ -19,6 +19,10 @@ export const createMediaTrack = (trackInit) => {
         (trackInit.kind === trackInit.input?.kind &&
             trackInit.kind === toMediaDeviceInputKind(trackInit.track));
     assert(trackConsistency, `Inconsistent track kind: ${trackInit.input?.kind} ${trackInit.track && toMediaDeviceInputKind(trackInit.track)} vs ${trackInit.kind}`);
+    assert(!hasOwn(trackInit, 'mute') ||
+        (hasOwn(trackInit, 'mute') && hasOwn(trackInit, 'overrideMute')), 'Inconsistent overrideMute mute()');
+    assert(!hasOwn(trackInit, 'muted') ||
+        (hasOwn(trackInit, 'muted') && hasOwn(trackInit, 'overrideMute')), 'Inconsistent overrideMute muted');
     const currentConstraints = typeof trackInit.constraints === 'boolean' ? {} : trackInit.constraints;
     // Find the original source track through the linked list
     let sourceMediaTrack = trackInit.previousMediaTrack;
@@ -41,6 +45,25 @@ export const createMediaTrack = (trackInit) => {
             ...(trackInit.getSettings?.() ?? {}),
         };
     };
+    // Suscribe track events
+    let trackUnsubscribe;
+    if (trackInit.track) {
+        trackUnsubscribe = createStreamTrackEventSubscriptions(trackInit.track, {
+            ended(track) {
+                assert(track.id === trackInit.track?.id, 'Same Track');
+                void mediaTrack.release();
+                trackInit.signals?.onMediaTrackStopped?.emit(mediaTrack);
+            },
+            mute(track) {
+                assert(track.id === trackInit.track?.id, 'Same Track');
+                trackInit.signals?.onMediaTrackSuspended?.emit(mediaTrack);
+            },
+            unmute(track) {
+                assert(track.id === trackInit.track?.id, 'Same Track');
+                trackInit.signals?.onMediaTrackResumed?.emit(mediaTrack);
+            },
+        });
+    }
     const mediaTrack = {
         get kind() {
             return trackInit.kind;
@@ -52,9 +75,10 @@ export const createMediaTrack = (trackInit) => {
             return trackInit.track;
         },
         get input() {
-            return trackInit.track?.readyState !== 'live'
-                ? undefined
-                : trackInit.input;
+            if (!trackInit.track) {
+                return undefined;
+            }
+            return trackInit.input;
         },
         get expectedInput() {
             return trackInit.expectedInput;
@@ -69,33 +93,53 @@ export const createMediaTrack = (trackInit) => {
             if (trackInit.overrideMute) {
                 return trackInit.muted;
             }
-            return (
-            // If the source track muted, the following processed track will
-            // be muted as well since there is no data to process
-            !!isTrackMuted(this.source.track) ||
-                isTrackMuted(trackInit.track));
+            if (trackInit.track) {
+                return !trackInit.track.enabled;
+            }
+            return undefined;
+        },
+        get stopped() {
+            return trackInit.track && isTrackEnded(trackInit.track);
+        },
+        get suspended() {
+            // Any track that is suspended in the chain will cause this track to be suspended
+            return (trackInit.previousMediaTrack?.suspended ||
+                trackInit.track?.muted);
         },
         get source() {
-            return sourceMediaTrack ?? mediaTrack;
+            return sourceMediaTrack ?? this;
         },
-        mute(toMute) {
+        mute(toMute, soft) {
+            if (this.muted === toMute) {
+                return;
+            }
             if (trackInit.overrideMute) {
-                return trackInit.mute?.(toMute);
+                trackInit.mute?.(toMute, this, soft);
             }
-            trackInit.previousMediaTrack?.mute(toMute);
-            if (trackInit.track) {
-                trackInit.track.enabled = !toMute;
+            else {
+                if (trackInit.track) {
+                    trackInit.track.enabled = !toMute;
+                }
+                trackInit.previousMediaTrack?.mute(toMute, soft);
             }
+            trackInit.signals?.onMediaTrackMuted?.emit(this);
         },
         getSettings,
         getConstraints() {
             return getConstraints();
         },
-        clone(signals) {
+        clone(signals, label) {
             return createMediaTrack({
-                ...trackInit,
+                kind: this.kind,
+                label: [label, this.label]
+                    .flatMap(a => (a ? [a] : []))
+                    .join('|'),
+                input: this.input,
                 constraints: getConstraints(),
-                track: trackInit.track?.clone(),
+                track: this.track?.clone(),
+                // When Cloning a MediaTrack, overrideMute should be turned off since
+                // it might corrupt the logic of its original media pipeline
+                overrideMute: false,
                 signals,
             });
         },
@@ -135,23 +179,20 @@ export const createMediaTrack = (trackInit) => {
             Object.assign(currentConstraints, resolvedConstraints);
         },
         async release() {
+            trackUnsubscribe?.();
             trackInit.track?.stop();
-            if (trackInit.track) {
-                trackInit.signals?.onTrackReleased?.emit(trackInit.track);
-                trackInit.signals?.[trackInit.track.kind === 'audio'
-                    ? 'onAudioMuteStateChanged'
-                    : 'onVideoMuteStateChanged']?.emit(undefined);
-            }
-            await trackInit.previousMediaTrack?.release();
             await trackInit.release?.();
+            await trackInit.previousMediaTrack?.release();
+            trackInit.signals?.onMediaTrackStopped?.emit(this);
         },
         toJSON() {
             return {
-                label: trackInit.label,
-                track: trackInit.track,
-                previousMediaTrack: trackInit.previousMediaTrack,
-                constraints: getConstraints(),
-                settings: getSettings(),
+                label: this.label,
+                track: this.track,
+                input: this.input,
+                previousMediaTrack: this.previousMediaTrack,
+                constraints: this.getConstraints(),
+                settings: this.getSettings(),
             };
         },
     };
@@ -161,7 +202,7 @@ export const createTrackProcessingPipeline = (processors) => async (track) => {
     let lastTrack = track;
     for (const process of processors) {
         if (lastTrack.track?.readyState !== 'live') {
-            return lastTrack;
+            continue;
         }
         lastTrack = await process(lastTrack);
     }
@@ -208,17 +249,22 @@ export const findExpectedInput = (devices, constraints, input, kind) => {
  * there is no track to check
  */
 export const isTrackMuted = (track) => {
-    if (!track || track.readyState === 'ended') {
+    if (!track) {
         return undefined;
     }
     return track.muted || !track.enabled;
 };
+export const isTrackEnded = (track) => track.readyState === 'ended';
 export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevicesChanged) => {
     const props = {
         originalConstraints: mediaInit.originalConstraints ?? mediaInit.constraints,
         status: mediaInit.status,
         devices: mediaInit.devices,
         stream: mediaInit.stream ?? new MediaStream(),
+        // `undefined` is used to avoid double state overriding,
+        // because 2 subsequent `susppend` events could invalidate the previous assignment
+        audioAlreadyMuted: undefined,
+        videoAlreadyMuted: undefined,
         audioTrack: mediaInit.tracks
             .flatMap(track => {
             return track.kind === 'audioinput' ? [track] : [];
@@ -228,74 +274,77 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             .flatMap(track => (track.kind === 'videoinput' ? [track] : []))
             .at(0),
     };
-    const subscribeTrackEvents = (track) => createStreamTrackEventSubscriptions(track, {
-        ended: track => {
-            mediaInit.signals?.[track.kind === 'audio'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(undefined);
-            mediaInit.signals?.onStreamTrackEndedFinal?.emit(track);
-        },
-        mute: track => {
-            mediaInit.signals?.[track.kind === 'audio'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(true);
-            mediaInit.signals?.onStreamTrackMuted?.emit(track);
-        },
-        unmute: track => {
-            mediaInit.signals?.[track.kind === 'audio'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(!track.enabled);
-            mediaInit.signals?.onStreamTrackUnmuted?.emit(track);
-        },
-    });
-    const subscribeTrackAndSourceTrackEvents = (track) => {
-        let trackUnsubscribe;
-        let sourceTrackUnsubscribe;
-        if (track.track) {
-            trackUnsubscribe = subscribeTrackEvents(track.track);
-            if (track.source.track && track.source.track !== track.track) {
-                sourceTrackUnsubscribe = subscribeTrackEvents(track.source.track);
-            }
-        }
-        return () => {
-            trackUnsubscribe?.();
-            sourceTrackUnsubscribe?.();
-        };
-    };
-    const trackSubscriptions = new Map(mediaInit.tracks.flatMap(track => {
-        if (track.track) {
-            return [
-                [track.track, subscribeTrackAndSourceTrackEvents(track)],
-            ];
-        }
-        return [];
-    }));
     const getConstraints = () => {
         const audio = props.audioTrack?.getConstraints() ?? mediaInit.constraints.audio;
         const video = props.videoTrack?.getConstraints() ?? mediaInit.constraints.video;
         return { audio: audio, video: video };
     };
-    let unsubscribe = onDevicesChanged.add(devices => {
+    const updateDevices = (devices) => {
         props.devices = devices;
-    });
-    const muteTrack = (track, mute) => {
-        const previousMuteState = track.muted;
-        const previousEnabledState = track.track?.enabled;
-        track.mute(mute);
-        const currentMuteState = track.muted;
-        const currentEnabledState = track.track?.enabled;
-        if (track.track &&
-            previousEnabledState !== undefined &&
-            currentEnabledState !== undefined &&
-            previousEnabledState !== currentEnabledState) {
-            mediaInit.signals?.onStreamTrackEnabled?.emit(track.track);
+    };
+    // Two-hand control to avoid leaking of the media
+    // When the track is suspended by the system, we also mute it if it is not already muted.
+    // As it is possible that the track could still flow data even if it is system muted
+    const handleSuspended = (track) => {
+        switch (track.kind) {
+            case 'audioinput': {
+                if (props.audioAlreadyMuted === undefined) {
+                    props.audioAlreadyMuted = props.audioTrack?.muted === true;
+                }
+                if (!props.audioAlreadyMuted) {
+                    props.audioTrack?.mute(true, true);
+                }
+                break;
+            }
+            case 'videoinput': {
+                if (props.videoAlreadyMuted === undefined) {
+                    props.videoAlreadyMuted = props.videoTrack?.muted === true;
+                }
+                if (!props.videoAlreadyMuted) {
+                    props.videoTrack?.mute(true, true);
+                }
+                break;
+            }
         }
-        if (previousMuteState !== currentMuteState) {
-            mediaInit.signals?.[track.kind === 'audioinput'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(currentMuteState);
+    };
+    // When the track is resumed by the system, we only unmute the track
+    // if an only if it is muted by the suspended event
+    const handleResumed = (track) => {
+        switch (track.kind) {
+            case 'audioinput': {
+                assert(props.audioAlreadyMuted !== undefined, 'Audio Resume should not be triggerred before Suspended');
+                if (props.audioTrack?.suspended === false) {
+                    if (!props.audioAlreadyMuted) {
+                        props.audioTrack.mute(false);
+                    }
+                    props.audioAlreadyMuted = undefined;
+                }
+                break;
+            }
+            case 'videoinput': {
+                assert(props.videoAlreadyMuted !== undefined, 'Video Resume should not be triggerred before Suspended');
+                if (props.videoTrack?.suspended === false) {
+                    if (!props.videoAlreadyMuted) {
+                        props.videoTrack.mute(false);
+                    }
+                    props.videoAlreadyMuted = undefined;
+                }
+                break;
+            }
         }
     };
+    const subscribe = () => [
+        onDevicesChanged.add(updateDevices),
+        mediaInit.signals?.onMediaTrackSuspended?.add(handleSuspended),
+        mediaInit.signals?.onMediaTrackResumed?.add(handleResumed),
+    ];
+    let subscriptions = subscribe();
+    const unsubscribe = () => {
+        for (const unsubscribe of subscriptions) {
+            unsubscribe?.();
+        }
+        subscriptions.length = 0;
+    };
     return {
         get id() {
             return props.stream.id;
@@ -360,16 +409,16 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
                     props.devices.size('videoinput') > 0));
         },
         muteAudio(mute) {
-            if (!props.audioTrack) {
+            if (!props.audioTrack || props.audioTrack.suspended) {
                 return;
             }
-            muteTrack(props.audioTrack, mute);
+            props.audioTrack.mute(mute);
         },
         muteVideo(mute) {
-            if (!props.videoTrack) {
+            if (!props.videoTrack || props.videoTrack.suspended) {
                 return;
             }
-            muteTrack(props.videoTrack, mute);
+            props.videoTrack?.mute(mute);
         },
         applyConstraints: async (constraints) => {
             await Promise.all([
@@ -382,12 +431,7 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             ]);
         },
         async release() {
-            unsubscribe?.();
-            unsubscribe = undefined;
-            for (const unsubscribe of trackSubscriptions.values()) {
-                unsubscribe();
-            }
-            trackSubscriptions.clear();
+            unsubscribe();
             await Promise.all([
                 props.audioTrack?.release(),
                 props.videoTrack?.release(),
@@ -397,26 +441,25 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             audio: props.audioTrack?.getSettings(),
             video: props.videoTrack?.getSettings(),
         }),
-        clone(signals) {
+        clone(signals, label) {
             const tracks = [props.audioTrack, props.videoTrack].flatMap(track => {
                 if (!track?.source) {
                     return [];
                 }
-                const cloned = track.source.clone(signals);
-                // Restore the enabled state for all cloned track
-                cloned.mute(false);
-                return cloned;
+                const cloned = track.source.clone(signals, label);
+                return [cloned];
             });
             const stream = this.stream &&
                 new MediaStream(tracks.flatMap(track => (track.track ? [track.track] : [])));
             const clonedMedia = buildMedia({
-                tracks,
-                devices: this.devices,
-                status: this.status,
-                stream,
                 constraints: this.getConstraints(),
+                devices: this.devices,
                 originalConstraints: this.getOriginalConstraints(),
                 permission: mediaInit.permission,
+                signals,
+                status: this.status,
+                stream,
+                tracks,
             });
             return clonedMedia;
         },
@@ -429,6 +472,9 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
         getVideoTracks() {
             return props.videoTrack ? [props.videoTrack] : [];
         },
+        isCurrentTrack(track) {
+            return track === props.audioTrack || track === props.videoTrack;
+        },
         addTrack(track) {
             switch (track.kind) {
                 case 'audioinput':
@@ -446,13 +492,22 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
                 default:
                     break;
             }
+            if (subscriptions.length === 0) {
+                subscriptions = subscribe();
+            }
             if (track.track) {
                 props.stream.addTrack(track.track);
-                if (!trackSubscriptions.has(track.track)) {
-                    trackSubscriptions.set(track.track, subscribeTrackAndSourceTrackEvents(track));
-                }
                 mediaInit.signals?.onAddTrack?.emit(track.track);
                 props.stream.dispatchEvent(new MediaStreamTrackEvent('addtrack', { track: track.track }));
+                if (track.muted) {
+                    mediaInit.signals?.onMediaTrackMuted?.emit(track);
+                }
+                if (track.suspended) {
+                    mediaInit.signals?.onMediaTrackSuspended?.emit(track);
+                }
+                if (track.stopped) {
+                    mediaInit.signals?.onMediaTrackStopped?.emit(track);
+                }
             }
         },
         removeTrack(track) {
@@ -467,13 +522,14 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             }
             if (removed && track.track) {
                 props.stream.removeTrack(track.track);
-                trackSubscriptions.get(track.track)?.();
-                trackSubscriptions.delete(track.track);
                 mediaInit.signals?.onRemoveTrack?.emit(track.track);
                 props.stream.dispatchEvent(new MediaStreamTrackEvent('removetrack', {
                     track: track.track,
                 }));
             }
+            if (subscriptions.length) {
+                unsubscribe();
+            }
         },
         toJSON() {
             return {
@@ -634,15 +690,20 @@ export const mergeSettings = (settingsA, settingsB) => {
  * @param percentage - The percentage of image height to calculate the blur
  * kernel size
  * @param height - The image height
+ * @param lowestProcessingHeight - The lowest image height that is processed by the consumer, used to adjust the blur kernel size across different resolutions
  * @param max - The upper bound
  *
  * @returns blur kernel size
  */
-export const getBlurKernelSize = (percentage, height, max = calculateMaxBlurPass(height)) => {
+export const getBlurKernelSize = (percentage, height, lowestProcessingHeight, max = calculateMaxBlurPass(height)) => {
     if (height <= 0 || percentage <= 0 || max <= 0) {
         return 0;
     }
-    return Math.min(Math.ceil(percentage * 0.01 * max), max);
+    const minHeightMaxBlurPass = lowestProcessingHeight
+        ? calculateMaxBlurPass(lowestProcessingHeight)
+        : max;
+    return Math.min(Math.ceil(percentage * 0.01 * max) +
+        Math.max(0, max - minHeightMaxBlurPass), max);
 };
 /**
  * Apply the content hint to the track

package/dist/videoProcessor.d.ts

@@ -1,10 +1,10 @@
-import type { VideoProcessor, SegmentationTransform, SegmentationModel, RenderBackend } from '@pexip/media-processor';
+import type { RenderBackend, SegmentationModel, SegmentationTransform, VideoProcessor } from '@pexip/media-processor';
 import type { MediaDeviceRequest } from '@pexip/media-control';
-import type { TrackProcessor, VideoRenderParams, Segmenters, VideoStreamTrackProcessorAPIs, VideoContentHint } from './types';
+import type { MediaSignals, Segmenters, TrackProcessor, VideoContentHint, VideoRenderParams, VideoStreamTrackProcessorAPIs } from './types';
 interface ProcessorDeps {
-    videoProcessor?: () => VideoProcessor;
-    transformer?: SegmentationTransform;
     segmenters: Partial<Segmenters>;
+    transformer?: SegmentationTransform;
+    videoProcessor?: () => VideoProcessor;
     videoSegmentationModel?: SegmentationModel;
 }
 interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<ProcessorDeps, 'videoProcessor'> {
@@ -18,14 +18,17 @@ interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<Pro
      * Whether or to enable this processor
      */
     shouldEnable: () => boolean;
+    lowestProcessingHeight?: number;
     processingWidth: number;
     processingHeight: number;
     hasInitializedDeps?: boolean;
     width?: number;
     height?: number;
     label?: string;
+    stopAsMute?: () => boolean;
     dynamicProcessingDimensions?: () => boolean;
     gpuAPI?: () => RenderBackend;
+    signals?: MediaSignals;
 }
 interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<ProcessorDeps> {
     hasInitialized: boolean;
@@ -35,5 +38,5 @@ declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUr
 type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
 type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest["video"], props: FeatureProps) => FeatureProps;
-export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, label, dynamicProcessingDimensions, gpuAPI, ...options }: VideoStreamProcessOptions) => TrackProcessor;
+export declare const createVideoStreamProcess: ({ backgroundImageUrl, dynamicProcessingDimensions, edgeBlurAmount, foregroundThreshold, frameRate, gpuAPI, label, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute, trackProcessorAPI, videoSegmentation, signals, ...options }: VideoStreamProcessOptions) => TrackProcessor;
 export {};