@pexip/media 20.2.0 → 20.3.0

package/CHANGELOG.md

@@ -1,5 +1,48 @@
 # @pexip/media
 
+## 20.3.0
+
+### Minor Changes
+
+- 69d6be4: Refactor `MediaTrack` to decouple system/hardware mute from mute
+
+  - Add `stopped` attributes to `MediaTrack` to indicate the track `live` or
+    `ended`
+  - Add `suspended` attributes to
+    `MediaTrack to indicate the track is being system/hardware muted`
+  - Add signal supports to `MediaTrack` to be able to subscribe track related
+    signals
+  - Implement Two-hand control to secure mute consistency
+
+    When there is `muted` event from `MediaStreamTrack`, the track will follow
+    the event to set the `enabled` attribute to `false` to avoid any
+    possibilities of leaking the local media to remote parties. When the
+    `unmuted` event is triggered, it depends on whether the `enabled` attribute
+    is set before the `muted` event or not to unmute the track.
+
+  Breaking changes:
+
+  - Replace `MediaStreamTrack` related signal with the new `MediaTrack` signals
+
+    - `onStreamTrackEnded` -> `onMediaTrackStopped`
+    - `onStreamTrackMuted` -> `onMediaTrackSuspended`
+    - `onStreamTrackUnmuted` -> `onMediaTrackResumed`
+    - `onStreamTrackEnabled` -> `onMediaTrackMuted`
+    - `onAudioMuteStateChanged` -> `onMediaTrackMuted`
+    - `onVideoMuteStateChanged` -> `onMediaTrackMuted`
+
+### Patch Changes
+
+- f8a8159: Fix Subsequant mute/unmute events handling
+- e347770: Handle suspended state fro preview controller
+- 0a375c7: Delay stopping video track to ensure last frame being black
+- Updated dependencies [69d6be4]
+- Updated dependencies [adc1a50]
+  - @pexip/media-control@20.3.0
+  - @pexip/utils@17.0.1
+  - @pexip/media-processor@20.3.0
+  - @pexip/signal@16.9.0
+
 ## 20.2.0
 
 ### Minor Changes

package/dist/audioMixingProcessor.d.ts

@@ -1,6 +1,6 @@
 import type { MediaDeviceRequest } from '@pexip/media-control';
 import type { AudioGraph, AudioNodeInit } from '@pexip/media-processor';
-import type { TrackProcessor } from './types';
+import type { MediaSignals, TrackProcessor } from './types';
 interface AudioStreamProcessorProps {
     mixWithAdditionalMedia?: boolean;
     merger?: AudioNodeInit<ChannelMergerNode, ChannelMergerNode>;
@@ -12,7 +12,7 @@ type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
 type FeatureProps = Pick<AudioStreamProcessorProps, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest["audio"], props: FeatureProps) => FeatureProps;
 /**
- * Create a Audio Mixing Processor and will own the stream passed-in
+ * Create an Audio Mixing Processor and will own the stream passed-in
  */
-export declare const createAudioMixingProcess: (getCurrentMedia: () => MediaStream | undefined, label?: "AudioMixingProcessor") => TrackProcessor;
+export declare const createAudioMixingProcess: (getCurrentMedia: () => MediaStream | undefined, signals?: MediaSignals, label?: "AudioMixingProcessor") => TrackProcessor;
 export {};

package/dist/audioMixingProcessor.js

@@ -2,7 +2,7 @@ import { extractConstraintsWithKeys } from '@pexip/media-control';
 import { isEmpty, assert } from '@pexip/utils';
 import { createAudioGraph, createAudioGraphProxy, createChannelMergerGraphNode, createStreamDestinationGraphNode, createStreamSourceGraphNode, resumeAudioOnUnmute, } from '@pexip/media-processor';
 import { logger } from './logger';
-import { createMediaTrack, isTrackMuted } from './utils';
+import { createMediaTrack } from './utils';
 import { PROCESSOR_LABELS } from './constants';
 const FEATURE_KEYS = ['mixWithAdditionalMedia'];
 const getAudioConstraints = extractConstraintsWithKeys(FEATURE_KEYS);
@@ -19,9 +19,9 @@ export const updateFeatureProps = (constraints, props) => {
     }, {});
 };
 /**
- * Create a Audio Mixing Processor and will own the stream passed-in
+ * Create an Audio Mixing Processor and will own the stream passed-in
  */
-export const createAudioMixingProcess = (getCurrentMedia, label = PROCESSOR_LABELS.AudioMixingProcessor) => {
+export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCESSOR_LABELS.AudioMixingProcessor) => {
     const props = {};
     return async (prevMediaTrack) => {
         updateFeatureProps(prevMediaTrack.getConstraints(), props);
@@ -154,12 +154,12 @@ export const createAudioMixingProcess = (getCurrentMedia, label = PROCESSOR_LABE
             expectedInput: prevMediaTrack.expectedInput,
             track,
             overrideMute: true,
-            mute: toMute => {
-                prevMediaTrack.mute(toMute);
+            mute: (toMute, _self, soft) => {
                 for (const track of mainSource.node?.mediaStream.getAudioTracks() ??
                     []) {
                     track.enabled = !toMute;
                 }
+                prevMediaTrack.mute(toMute, soft);
             },
             get muted() {
                 return (
@@ -168,11 +168,12 @@ export const createAudioMixingProcess = (getCurrentMedia, label = PROCESSOR_LABE
                 !!prevMediaTrack.muted ||
                     mainSource.node?.mediaStream
                         .getAudioTracks()
-                        .some(isTrackMuted));
+                        .some(track => track.enabled === false));
             },
             constraints: prevMediaTrack.getConstraints(),
             applyConstraints,
             release,
+            signals,
             getSettings: () => {
                 const mixWithAdditionalMedia = !!props.mixWithAdditionalMedia;
                 const contentHint = track.contentHint ?? '';

package/dist/audioProcessor.d.ts

@@ -1,6 +1,6 @@
-import type { AudioGraphOptions, AudioNodeInit, ThrottleOptions, DenoiseWorkletNodeInit, AnalyzerNodeInit, AudioGraph } from '@pexip/media-processor';
+import type { AnalyzerNodeInit, AudioGraph, AudioGraphOptions, AudioNodeInit, DenoiseWorkletNodeInit, ThrottleOptions } from '@pexip/media-processor';
 import type { MediaDeviceRequest } from '@pexip/media-control';
-import type { TrackProcessor, DenoiseParams, AudioContentHint } from './types';
+import type { AudioContentHint, DenoiseParams, MediaSignals, TrackProcessor } from './types';
 type AudioNodeInits = AudioNodeInit[];
 /**
  * A function to be called to create the AudioNodes needed for the graph
@@ -60,6 +60,7 @@ interface AudioProcessOptions {
      */
     silentThreshold?: number;
     label?: string;
+    signals?: MediaSignals;
 }
 interface AudioStreamProcessorProps {
     audioGraphOptions?: AudioGraphOptions;
@@ -80,7 +81,7 @@ type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
 type FeatureProps = Pick<AudioStreamProcessorProps, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest["audio"], props: FeatureProps) => FeatureProps;
 /**
- * Create a Audio Stream Processor and will own the stream passed-in
+ * Create an Audio Stream Processor and will own the stream passed-in
  */
-export declare const createAudioStreamProcess: ({ analyzerUpdateFrequency, audioGraphOptions, audioSignalDetectionDuration, clock, createNodes, denoiseParams, fftSize, onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold, throttleMs, label, }: AudioProcessOptions & ThrottleOptions) => TrackProcessor;
+export declare const createAudioStreamProcess: ({ analyzerUpdateFrequency, audioGraphOptions, audioSignalDetectionDuration, clock, createNodes, denoiseParams, fftSize, onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, signals, silentThreshold, throttleMs, label, }: AudioProcessOptions & ThrottleOptions) => TrackProcessor;
 export {};

package/dist/audioProcessor.js

@@ -1,6 +1,6 @@
 import { extractConstraintsWithKeys } from '@pexip/media-control';
 import { createQueue, isEmpty, assert } from '@pexip/utils';
-import { createAudioGraph, createAudioGraphProxy, createStreamSourceGraphNode, createStreamDestinationGraphNode, createAnalyzerSubscribableGraphNode, createDenoiseWorkletGraphNode, createAudioSignalDetector, createVADetector, createVoiceDetectorFromTimeData, createVoiceDetectorFromProbability, avg, } from '@pexip/media-processor';
+import { avg, createAnalyzerSubscribableGraphNode, createAudioGraph, createAudioGraphProxy, createAudioSignalDetector, createDenoiseWorkletGraphNode, createStreamDestinationGraphNode, createStreamSourceGraphNode, createVADetector, createVoiceDetectorFromProbability, createVoiceDetectorFromTimeData, } from '@pexip/media-processor';
 import { PROCESSOR_LABELS } from './constants';
 import { logger } from './logger';
 import { isAudioContentHint } from './typeGuard';
@@ -54,12 +54,12 @@ export const updateFeatureProps = (constraints, props) => {
     }, {});
 };
 /**
- * Create a Audio Stream Processor and will own the stream passed-in
+ * Create an Audio Stream Processor and will own the stream passed-in
  */
 export const createAudioStreamProcess = ({ analyzerUpdateFrequency = 0.5, // 0.5 Hz
 audioGraphOptions, audioSignalDetectionDuration = 4.0, // 4 seconds
 clock, createNodes, denoiseParams, fftSize = 2048, // FFT size
-onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold = 10.0 / 32767, // At least one LSB 16-bit data (compare is on absolute value).
+onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, signals, silentThreshold = 10.0 / 32767, // At least one LSB 16-bit data (compare is on absolute value).
 throttleMs = 3000, // 3 seconds
 label = PROCESSOR_LABELS.AudioProcessor, }) => {
     const props = {
@@ -198,11 +198,6 @@ label = PROCESSOR_LABELS.AudioProcessor, }) => {
             props.analyzer = undefined;
             props.audioGraph = undefined;
         };
-        const mute = (mute) => {
-            if (track) {
-                track.enabled = !mute;
-            }
-        };
         return createMediaTrack({
             label,
             kind: 'audioinput',
@@ -210,9 +205,18 @@ label = PROCESSOR_LABELS.AudioProcessor, }) => {
             previousMediaTrack: prevMediaTrack,
             input: prevMediaTrack.input,
             expectedInput: prevMediaTrack.expectedInput,
-            mute,
+            overrideMute: true,
+            mute(toMute) {
+                if (track) {
+                    track.enabled = !toMute;
+                }
+            },
+            get muted() {
+                return !track.enabled;
+            },
             track,
             release,
+            signals,
             applyConstraints: async (constraints) => {
                 if (isEmpty(constraints)) {
                     return;

package/dist/media.js

@@ -1,4 +1,4 @@
-import { createIndexedDevices, createStreamTrackEventSubscriptions, getDevices, getInputDevicePermissionState, isIndexedDevices, isRequestedResolution, mergeConstraints, } from '@pexip/media-control';
+import { createIndexedDevices, getDevices, getInputDevicePermissionState, isIndexedDevices, isRequestedResolution, mergeConstraints, } from '@pexip/media-control';
 import { createAsyncQueue, isEmpty, assert } from '@pexip/utils';
 import { internalSignals } from './signals';
 import { UserMediaStatus } from './types';
@@ -283,28 +283,6 @@ export const createMedia = ({ getMuteState, signals, audioProcessors, videoProce
         props.devices = createIndexedDevices(await getDevices());
         return props.devices;
     };
-    const syncMuteState = (tracks) => {
-        const inputMuted = getMuteState();
-        for (const track of tracks) {
-            switch (track.kind) {
-                case 'audioinput': {
-                    track.mute(inputMuted.audio);
-                    // FIXME: this can fire too often, but we want to make sure we signal initial state as well.
-                    // We should only fire this when needed.
-                    signals.onAudioMuteStateChanged?.emit(track.muted);
-                    break;
-                }
-                case 'videoinput':
-                    track.mute(inputMuted.video);
-                    // FIXME: this can fire too often, but we want to make sure we signal initial state as well.
-                    // We should only fire this when needed.
-                    signals.onVideoMuteStateChanged?.emit(track.muted);
-                    break;
-                default:
-                    break;
-            }
-        }
-    };
     const processMedia = createMediaProcessor({
         audioProcessors,
         videoProcessors,
@@ -332,13 +310,14 @@ export const createMedia = ({ getMuteState, signals, audioProcessors, videoProce
     });
     const processAndUpdateMedia = async (media, tracks) => {
         const processedTracks = await processMedia(tracks);
-        // Sync mute state to processed tracks
-        syncMuteState(processedTracks);
         for (const [idx, track] of processedTracks.entries()) {
             const originTrack = tracks.at(idx);
             assert(originTrack, 'Processed track should always has the original track in the same order');
             // Only replace track when they are not the same
             if (originTrack.id !== track.id) {
+                if (originTrack.muted !== undefined) {
+                    track.mute(originTrack.muted);
+                }
                 media.removeTrack(originTrack);
                 media.addTrack(track);
             }
@@ -351,19 +330,12 @@ export const createMedia = ({ getMuteState, signals, audioProcessors, videoProce
         shouldDiscardMedia: () => props.discardMedia,
         getDefaultConstraints,
         onMediaTracksChanged: (media, tracks) => {
-            // Sync mute state to input tracks
-            syncMuteState(tracks);
+            const inputMuted = getMuteState();
             for (const track of tracks) {
-                if (track.track) {
-                    const unsubscribe = createStreamTrackEventSubscriptions(track.track, {
-                        ended: track => {
-                            signals.onStreamTrackEnded?.emit(track);
-                            unsubscribe();
-                        },
-                        mute: signals.onStreamTrackMuted?.emit,
-                        unmute: signals.onStreamTrackUnmuted?.emit,
-                    });
-                }
+                // Sync mute state to input tracks
+                track.mute(track.kind === 'audioinput'
+                    ? inputMuted.audio
+                    : inputMuted.video);
             }
             logger.debug({ media, tracks }, 'Update media tracks');
             props.media = media;

package/dist/previewController.d.ts

@@ -12,11 +12,13 @@ export interface PreviewEventHandler {
     audioInputError?: EventErrorCallback;
     applyChangesError?: EventErrorCallback;
     revertChangesError?: EventErrorCallback;
+    audioMuted?: EventCallback<boolean | undefined>;
+    audioSuspended?: EventCallback<boolean | undefined>;
+    updatingMain?: EventCallback<boolean>;
     updatingPreviewAudio?: EventCallback<boolean>;
     updatingPreviewVideo?: EventCallback<boolean>;
-    audioMuted?: EventCallback<boolean | undefined>;
     videoMuted?: EventCallback<boolean | undefined>;
-    updatingMain?: EventCallback<boolean>;
+    videoSuspended?: EventCallback<boolean | undefined>;
     unsubscribeMain?: Unsubscribe;
 }
 export interface PreviewStreamParams {
@@ -24,6 +26,7 @@ export interface PreviewStreamParams {
     getCurrentMedia: () => Media | undefined;
     updateMainStream: (request: MediaDeviceRequest) => Promise<void>;
     mainMediaSignal: MediaSignals['onMediaChanged'];
+    signals: MediaSignals;
     onEnded?: () => void;
     fftSize?: number;
     queueOptions?: Partial<AsyncQueueOptions>;
@@ -76,8 +79,10 @@ export interface PreviewStreamController {
     onApplyChangesError(callback: EventErrorCallback): Unsubscribe;
     onRevertChangesError(callback: EventErrorCallback): Unsubscribe;
     onAudioMuted(callback: EventCallback<boolean | undefined>): Unsubscribe;
+    onAudioSuspended(callback: EventCallback<boolean | undefined>): Unsubscribe;
     onVideoMuted(callback: EventCallback<boolean | undefined>): Unsubscribe;
+    onVideoSuspended(callback: EventCallback<boolean | undefined>): Unsubscribe;
 }
-export declare const createPreviewStreamController: ({ getCurrentDevices, getCurrentMedia, updateMainStream, onEnded, mainMediaSignal, queueOptions, audioProcessors, videoProcessors, getDefaultConstraints, }: PreviewStreamParams) => PreviewStreamController;
+export declare const createPreviewStreamController: ({ getCurrentDevices, getCurrentMedia, updateMainStream, onEnded, mainMediaSignal, queueOptions, audioProcessors, videoProcessors, getDefaultConstraints, signals, }: PreviewStreamParams) => PreviewStreamController;
 export type CreatePreviewStreamController = typeof createPreviewStreamController;
 export {};

package/dist/previewController.js

@@ -4,7 +4,6 @@ import { AUDIO_SETTINGS_KEYS, VIDEO_SETTINGS_KEYS, createMediaProcessor, hasSett
 import { isMedia } from './typeGuard';
 import { createModuleLogger } from './logger';
 import { createGetUserMediaProcess, requestUserMediaWithRetry, } from './userMedia';
-import { createMediaSignals } from './signals';
 import { createMediaUpdater } from './media';
 const DEFAULT_QUEUE_DELAY_MS = 100;
 const DEFAULT_QUEUE_DROP_LAST = false;
@@ -52,7 +51,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
     throttleInMS: DEFAULT_QUEUE_THROTTLE_MS,
     delayInMS: DEFAULT_QUEUE_DELAY_MS,
     dropLast: DEFAULT_QUEUE_DROP_LAST,
-}, audioProcessors, videoProcessors, getDefaultConstraints = () => ({}), }) => {
+}, audioProcessors, videoProcessors, getDefaultConstraints = () => ({}), signals, }) => {
     const queue = createAsyncQueue(queueOptions);
     const eventHandlers = {};
     const internalProps = {
@@ -62,15 +61,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
         updatingMain: false,
         discardMedia: false,
         initialized: false,
-        signals: createMediaSignals([
-            'onAddTrack',
-            'onAudioMuteStateChanged',
-            'onRemoveTrack',
-            'onStatusChanged',
-            'onUpdatingAudio',
-            'onUpdatingVideo',
-            'onVideoMuteStateChanged',
-        ], 'PreviewStreamController'),
+        signals,
     };
     const logger = createModuleLogger({
         module: 'PreviewStreamController',
@@ -129,11 +120,29 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
         },
     });
     const subscriptions = [
-        props.signals.onAudioMuteStateChanged?.add(muted => {
-            eventHandlers.audioMuted?.(muted);
+        props.signals.onMediaTrackMuted?.add(track => {
+            if (track.kind === 'audioinput') {
+                eventHandlers.audioMuted?.(track.muted);
+            }
+            else {
+                eventHandlers.videoMuted?.(track.muted);
+            }
+        }),
+        props.signals.onMediaTrackSuspended?.add(track => {
+            if (track.kind === 'audioinput') {
+                eventHandlers.audioSuspended?.(track.suspended);
+            }
+            else {
+                eventHandlers.videoSuspended?.(track.suspended);
+            }
         }),
-        props.signals.onVideoMuteStateChanged?.add(muted => {
-            eventHandlers.videoMuted?.(muted);
+        props.signals.onMediaTrackResumed?.add(track => {
+            if (track.kind === 'audioinput') {
+                eventHandlers.audioSuspended?.(track.suspended);
+            }
+            else {
+                eventHandlers.videoSuspended?.(track.suspended);
+            }
         }),
         props.signals.onAddTrack?.add(track => {
             switch (track.kind) {
@@ -228,7 +237,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
             return;
         }
         try {
-            const clonedMedia = mainMedia.clone(props.signals);
+            const clonedMedia = mainMedia.clone(props.signals, 'Preview');
             props.media = clonedMedia;
             queue.enqueue(async () => {
                 await processAndUpdateMedia(clonedMedia, clonedMedia.getTracks(), true);
@@ -484,6 +493,8 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
         onUpdatingMain: toEvenHandler('updatingMain'),
         onAudioMuted: toEvenHandler('audioMuted'),
         onVideoMuted: toEvenHandler('videoMuted'),
+        onAudioSuspended: toEvenHandler('audioSuspended'),
+        onVideoSuspended: toEvenHandler('videoSuspended'),
         applyChanges,
         revertChanges,
         cleanup,

package/dist/types.d.ts

@@ -127,8 +127,9 @@ export interface Media extends MediaAttributes {
     getAudioTracks(): MediaTrack[];
     getVideoTracks(): MediaTrack[];
     addTrack(track: MediaTrack): void;
+    isCurrentTrack(track: MediaTrack): boolean;
     removeTrack(track: MediaTrack): void;
-    clone(signals?: MediaSignals): Media;
+    clone(signals?: MediaSignals, label?: string): Media;
 }
 export interface MediaInit {
     constraints: MediaDeviceRequest;
@@ -149,26 +150,35 @@ export interface MediaTrack {
      * `track` is available otherwise it uses the `id` from the construction or the
      * `kind`. @see MediaTrackInit
      */
-    id: string;
-    stopped: boolean;
-    kind: 'audioinput' | 'videoinput';
-    label?: string;
-    previousMediaTrack?: MediaTrack;
+    readonly id: string;
+    /**
+     * It maps to MediaStreamTrack['readyState']
+     * When `true` the track's `readyState` is `ended`, `false` means `live` otherwise the track is not available
+     */
+    readonly stopped?: boolean;
+    /**
+     * Indicate that the media has been suspended by the User Agent, and it could happen at any time.
+     * `undefined` means the track is not available
+     */
+    readonly suspended?: boolean;
+    readonly kind: 'audioinput' | 'videoinput';
+    readonly label?: string;
+    readonly previousMediaTrack?: MediaTrack;
     /**
      * The MediaStreamTrack object representing the track
      * @see https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack
      */
-    track?: MediaStreamTrack;
+    readonly track?: MediaStreamTrack;
     /**
      * The `MediaDeviceInfo` object representing the device that the track is
      * connected to. @see https://developer.mozilla.org/en-US/docs/Web/API/MediaDeviceInfo
      */
-    input?: MediaDeviceInfoLike;
+    readonly input?: MediaDeviceInfoLike;
     /**
      * The `MediaDeviceInfo` object representing the device that the track is
      * expected to be connected to. @see https://developer.mozilla.org/en-US/docs/Web/API/MediaDeviceInfo
      */
-    expectedInput?: MediaDeviceInfoLike;
+    readonly expectedInput?: MediaDeviceInfoLike;
     /**
      * The current mute state of the track. `undefined` means the track is not available
      */
@@ -176,13 +186,20 @@ export interface MediaTrack {
     /**
      * The source track that it is originated from
      */
-    source: MediaTrack;
-    mute(toMute: boolean): void;
+    readonly source: MediaTrack;
+    /**
+     * Mute the track
+     *
+     * @param toMute - `true` to mute the track, `false` to unmute
+     * @param soft - `true` to indicate a soft mute i.e. setting the track to NOT `enabled`
+     * `false` or `undefined` for default mute
+     */
+    mute(toMute: boolean, soft?: boolean): void;
     getSettings(): ExtendedMediaTrackSettings;
     getConstraints(): InputDeviceConstraint;
     applyConstraints(constraints: InputDeviceConstraint): Promise<void>;
     release(): Promise<void>;
-    clone(signals?: MediaSignals): MediaTrack;
+    clone(signals?: MediaSignals, label?: string): MediaTrack;
     toJSON(): unknown;
 }
 export interface MediaTrackInit extends Partial<Omit<MediaTrack, 'mute'>> {
@@ -192,7 +209,15 @@ export interface MediaTrackInit extends Partial<Omit<MediaTrack, 'mute'>> {
     constraints: InputDeviceConstraint;
     overrideMute?: boolean;
     signals?: MediaSignals;
-    mute?: (toMute: boolean, self: MediaTrack) => void;
+    /**
+     * Overriding Mute the track
+     *
+     * @param toMute - `true` to mute the track, `false` to unmute
+     * @param self - The media track itself
+     * @param soft - `true` to indicate a soft mute i.e. setting the track to NOT `enabled`
+     * `false` or `undefined` for default mute
+     */
+    mute?: (toMute: boolean, self: MediaTrack, soft?: boolean) => void;
 }
 export type Process<T> = (a: T) => Promise<MediaTrack>;
 export type TrackProcessor = Process<MediaTrack>;
@@ -521,44 +546,16 @@ export type VideoRenderParams = Omit<RendererOptions, 'effects'> & {
     tilt?: boolean;
     zoom?: boolean;
 };
-export interface StreamTrackSignals {
-    /**
-     * MediaStreamTrack events: mute
-     * https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
-     */
-    onStreamTrackMuted: Signal<MediaStreamTrack>;
-    /**
-     * MediaStreamTrack events: unmute
-     * https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
-     */
-    onStreamTrackUnmuted: Signal<MediaStreamTrack>;
-    /**
-     * MediaStreamTrack events: ended
-     * https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
-     */
-    onStreamTrackEnded: Signal<MediaStreamTrack>;
-    /**
-     * Emit `MediaStreamTrack` whenever a call to `Media['muteAudio']` or
-     * `Media['muteVideo']`
-     */
-    onStreamTrackEnabled: Signal<MediaStreamTrack>;
-}
-/**
- * Track state signals that used for the final track that the media pipeline
- * produced
- */
-export type StreamTrackSignalFinals = {
-    [key in keyof StreamTrackSignals as `${key}Final`]: Signal<MediaStreamTrack>;
-};
 export interface MediaChangesSignals {
     onAddTrack: Signal<MediaStreamTrack>;
-    onAudioMuteStateChanged: Signal<boolean | undefined>;
-    onTrackReleased: Signal<MediaStreamTrack>;
     onDevicesChanged: Signal<IndexedDevices>;
     onMediaChanged: Signal<Media | undefined>;
+    onMediaTrackMuted: Signal<MediaTrack>;
+    onMediaTrackResumed: Signal<MediaTrack>;
+    onMediaTrackStopped: Signal<MediaTrack>;
+    onMediaTrackSuspended: Signal<MediaTrack>;
     onRemoveTrack: Signal<MediaStreamTrack>;
     onStatusChanged: Signal<UserMediaStatus>;
-    onVideoMuteStateChanged: Signal<boolean | undefined>;
     onUpdatingAudio: Signal<boolean>;
     onUpdatingVideo: Signal<boolean>;
 }
@@ -566,7 +563,7 @@ export interface AudioDetectionSignals {
     onVAD: Signal<undefined>;
     onSilentDetected: Signal<boolean>;
 }
-export type MediaSignalsOptional = Omit<Partial<MediaChangesSignals>, 'onMediaChanged'> & Partial<StreamTrackSignals> & Partial<StreamTrackSignalFinals>;
+export type MediaSignalsOptional = Omit<Partial<MediaChangesSignals>, 'onMediaChanged'>;
 export type MediaSignalsRequired = Pick<MediaChangesSignals, 'onMediaChanged'> & AudioDetectionSignals;
 export type MediaSignals = MediaSignalsRequired & MediaSignalsOptional;
 export declare enum DeniedDevices {

package/dist/userMedia.js

@@ -1,7 +1,7 @@
 import { MediaDeviceFailure, extractConstraintsWithKeys, findMediaInputFromMediaStreamTrack, getUserMedia, isExactDeviceConstraint, isStreamingRequestedDevices, relaxInputConstraint, } from '@pexip/media-control';
 import { assert } from '@pexip/utils';
 import { UserMediaStatus } from './types';
-import { buildMedia, createMediaTrack, findExpectedInput, isTrackMuted, makeDeriveDeviceStatus, } from './utils';
+import { buildMedia, createMediaTrack, findExpectedInput, makeDeriveDeviceStatus, } from './utils';
 import { logger } from './logger';
 import { isUnknownError } from './status';
 import { PROCESSOR_LABELS } from './constants';
@@ -556,20 +556,33 @@ export const createGetUserMediaProcess = ({ getCurrentDevices, getUserMedia, sco
         let audioMediaTrack;
         let videoMediaTrack;
         const extractContentHint = extractConstraintsWithKeys(['contentHint']);
+        let trackReleaseTimeoutID = 0;
         /**
          * Release the track as mute
          */
-        const releaseTrackAsMute = async (track, toMute) => {
+        const releaseTrackAsMute = async (track, toMute, soft) => {
             if (!track?.track || track.muted === toMute) {
                 return;
             }
             track.track.enabled = !toMute;
+            if (soft) {
+                return;
+            }
             if (toMute) {
                 if (!track.stopped) {
-                    await track.release();
+                    // Track cannot be stopped immediately as we need to ensure
+                    // the last frame is black to avoid frozen frame in a gateway call
+                    trackReleaseTimeoutID = window.setTimeout(() => {
+                        track.release();
+                        trackReleaseTimeoutID = 0;
+                    }, 500);
                 }
             }
             else {
+                if (trackReleaseTimeoutID) {
+                    window.clearTimeout(trackReleaseTimeoutID);
+                    trackReleaseTimeoutID = 0;
+                }
                 if (track.stopped) {
                     await updateMedia({
                         [track.kind === 'audioinput' ? 'audio' : 'video']: track.getConstraints(),
@@ -586,7 +599,7 @@ export const createGetUserMediaProcess = ({ getCurrentDevices, getUserMedia, sco
                 audioTrack.contentHint = contentHint ?? '';
             }
             audioMediaTrack = createMediaTrack({
-                label: PROCESSOR_LABELS.GetUserMedia,
+                label: [scope, PROCESSOR_LABELS.GetUserMedia].join('|'),
                 kind: 'audioinput',
                 track: audioTrack,
                 input: audioInput,
@@ -605,17 +618,19 @@ export const createGetUserMediaProcess = ({ getCurrentDevices, getUserMedia, sco
                 videoTrack.contentHint = contentHint ?? '';
             }
             videoMediaTrack = createMediaTrack({
-                label: PROCESSOR_LABELS.GetUserMedia,
+                label: [scope, PROCESSOR_LABELS.GetUserMedia].join('|'),
                 kind: 'videoinput',
                 track: videoTrack,
                 input: videoInput,
                 expectedInput: findExpectedInput(devices, constraints.video, videoInput, 'videoinput'),
-                overrideMute: stopVideoTrackAsMute(),
+                get overrideMute() {
+                    return stopVideoTrackAsMute();
+                },
                 get muted() {
-                    return isTrackMuted(videoTrack);
+                    return !videoTrack?.enabled;
                 },
-                mute(toMute, self) {
-                    releaseTrackAsMute(self, toMute);
+                mute(toMute, self, soft) {
+                    releaseTrackAsMute(self, toMute, soft);
                 },
                 constraints: constraints.video,
                 signals,

package/dist/utils.js

@@ -19,10 +19,10 @@ export const createMediaTrack = (trackInit) => {
         (trackInit.kind === trackInit.input?.kind &&
             trackInit.kind === toMediaDeviceInputKind(trackInit.track));
     assert(trackConsistency, `Inconsistent track kind: ${trackInit.input?.kind} ${trackInit.track && toMediaDeviceInputKind(trackInit.track)} vs ${trackInit.kind}`);
-    assert(!trackInit.overrideMute ||
-        (trackInit.overrideMute &&
-            Object.hasOwn(trackInit, 'mute') &&
-            Object.hasOwn(trackInit, 'muted')), 'Inconsistent overrideMute');
+    assert(!hasOwn(trackInit, 'mute') ||
+        (hasOwn(trackInit, 'mute') && hasOwn(trackInit, 'overrideMute')), 'Inconsistent overrideMute mute()');
+    assert(!hasOwn(trackInit, 'muted') ||
+        (hasOwn(trackInit, 'muted') && hasOwn(trackInit, 'overrideMute')), 'Inconsistent overrideMute muted');
     const currentConstraints = typeof trackInit.constraints === 'boolean' ? {} : trackInit.constraints;
     // Find the original source track through the linked list
     let sourceMediaTrack = trackInit.previousMediaTrack;
@@ -45,6 +45,25 @@ export const createMediaTrack = (trackInit) => {
             ...(trackInit.getSettings?.() ?? {}),
         };
     };
+    // Suscribe track events
+    let trackUnsubscribe;
+    if (trackInit.track) {
+        trackUnsubscribe = createStreamTrackEventSubscriptions(trackInit.track, {
+            ended(track) {
+                assert(track.id === trackInit.track?.id, 'Same Track');
+                void mediaTrack.release();
+                trackInit.signals?.onMediaTrackStopped?.emit(mediaTrack);
+            },
+            mute(track) {
+                assert(track.id === trackInit.track?.id, 'Same Track');
+                trackInit.signals?.onMediaTrackSuspended?.emit(mediaTrack);
+            },
+            unmute(track) {
+                assert(track.id === trackInit.track?.id, 'Same Track');
+                trackInit.signals?.onMediaTrackResumed?.emit(mediaTrack);
+            },
+        });
+    }
     const mediaTrack = {
         get kind() {
             return trackInit.kind;
@@ -74,36 +93,50 @@ export const createMediaTrack = (trackInit) => {
             if (trackInit.overrideMute) {
                 return trackInit.muted;
             }
-            // The source is itself, just access the track's state
-            if (this.source === this) {
-                return isTrackMuted(trackInit.track);
+            if (trackInit.track) {
+                return !trackInit.track.enabled;
             }
-            return this.source.muted;
+            return undefined;
         },
         get stopped() {
-            return trackInit.track ? isTrackEnded(trackInit.track) : true;
+            return trackInit.track && isTrackEnded(trackInit.track);
+        },
+        get suspended() {
+            // Any track that is suspended in the chain will cause this track to be suspended
+            return (trackInit.previousMediaTrack?.suspended ||
+                trackInit.track?.muted);
         },
         get source() {
-            return sourceMediaTrack ?? mediaTrack;
+            return sourceMediaTrack ?? this;
         },
-        mute(toMute) {
+        mute(toMute, soft) {
+            if (this.muted === toMute) {
+                return;
+            }
             if (trackInit.overrideMute) {
-                return trackInit.mute?.(toMute, this);
+                trackInit.mute?.(toMute, this, soft);
             }
-            if (trackInit.track) {
-                trackInit.track.enabled = !toMute;
+            else {
+                if (trackInit.track) {
+                    trackInit.track.enabled = !toMute;
+                }
+                trackInit.previousMediaTrack?.mute(toMute, soft);
             }
-            trackInit.previousMediaTrack?.mute(toMute);
+            trackInit.signals?.onMediaTrackMuted?.emit(this);
         },
         getSettings,
         getConstraints() {
             return getConstraints();
         },
-        clone(signals) {
+        clone(signals, label) {
             return createMediaTrack({
-                ...trackInit,
+                kind: this.kind,
+                label: [label, this.label]
+                    .flatMap(a => (a ? [a] : []))
+                    .join('|'),
+                input: this.input,
                 constraints: getConstraints(),
-                track: trackInit.track?.clone(),
+                track: this.track?.clone(),
                 // When Cloning a MediaTrack, overrideMute should be turned off since
                 // it might corrupt the logic of its original media pipeline
                 overrideMute: false,
@@ -146,23 +179,20 @@ export const createMediaTrack = (trackInit) => {
             Object.assign(currentConstraints, resolvedConstraints);
         },
         async release() {
+            trackUnsubscribe?.();
             trackInit.track?.stop();
-            if (trackInit.track) {
-                trackInit.signals?.onTrackReleased?.emit(trackInit.track);
-                trackInit.signals?.[trackInit.track.kind === 'audio'
-                    ? 'onAudioMuteStateChanged'
-                    : 'onVideoMuteStateChanged']?.emit(undefined);
-            }
-            await trackInit.previousMediaTrack?.release();
             await trackInit.release?.();
+            await trackInit.previousMediaTrack?.release();
+            trackInit.signals?.onMediaTrackStopped?.emit(this);
         },
         toJSON() {
             return {
-                label: trackInit.label,
-                track: trackInit.track,
-                previousMediaTrack: trackInit.previousMediaTrack,
-                constraints: getConstraints(),
-                settings: getSettings(),
+                label: this.label,
+                track: this.track,
+                input: this.input,
+                previousMediaTrack: this.previousMediaTrack,
+                constraints: this.getConstraints(),
+                settings: this.getSettings(),
             };
         },
     };
@@ -231,6 +261,10 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
         status: mediaInit.status,
         devices: mediaInit.devices,
         stream: mediaInit.stream ?? new MediaStream(),
+        // `undefined` is used to avoid double state overriding,
+        // because 2 subsequent `susppend` events could invalidate the previous assignment
+        audioAlreadyMuted: undefined,
+        videoAlreadyMuted: undefined,
         audioTrack: mediaInit.tracks
             .flatMap(track => {
             return track.kind === 'audioinput' ? [track] : [];
@@ -240,73 +274,76 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             .flatMap(track => (track.kind === 'videoinput' ? [track] : []))
             .at(0),
     };
-    const subscribeTrackEvents = (track) => createStreamTrackEventSubscriptions(track, {
-        ended: track => {
-            mediaInit.signals?.[track.kind === 'audio'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(undefined);
-            mediaInit.signals?.onStreamTrackEndedFinal?.emit(track);
-        },
-        mute: track => {
-            mediaInit.signals?.[track.kind === 'audio'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(true);
-            mediaInit.signals?.onStreamTrackMuted?.emit(track);
-        },
-        unmute: track => {
-            mediaInit.signals?.[track.kind === 'audio'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(!track.enabled);
-            mediaInit.signals?.onStreamTrackUnmuted?.emit(track);
-        },
-    });
-    const subscribeTrackAndSourceTrackEvents = (track) => {
-        let trackUnsubscribe;
-        let sourceTrackUnsubscribe;
-        if (track.track) {
-            trackUnsubscribe = subscribeTrackEvents(track.track);
-            if (track.source.track && track.source.track !== track.track) {
-                sourceTrackUnsubscribe = subscribeTrackEvents(track.source.track);
-            }
-        }
-        return () => {
-            trackUnsubscribe?.();
-            sourceTrackUnsubscribe?.();
-        };
-    };
-    const trackSubscriptions = new Map(mediaInit.tracks.flatMap(track => {
-        if (track.track) {
-            return [
-                [track.track, subscribeTrackAndSourceTrackEvents(track)],
-            ];
-        }
-        return [];
-    }));
     const getConstraints = () => {
         const audio = props.audioTrack?.getConstraints() ?? mediaInit.constraints.audio;
         const video = props.videoTrack?.getConstraints() ?? mediaInit.constraints.video;
         return { audio: audio, video: video };
     };
-    let unsubscribe = onDevicesChanged.add(devices => {
+    const updateDevices = (devices) => {
         props.devices = devices;
-    });
-    const muteTrack = (track, mute) => {
-        const previousMuteState = track.muted;
-        const previousEnabledState = track.track?.enabled;
-        track.mute(mute);
-        const currentMuteState = track.muted;
-        const currentEnabledState = track.track?.enabled;
-        if (track.track &&
-            previousEnabledState !== undefined &&
-            currentEnabledState !== undefined &&
-            previousEnabledState !== currentEnabledState) {
-            mediaInit.signals?.onStreamTrackEnabled?.emit(track.track);
+    };
+    // Two-hand control to avoid leaking of the media
+    // When the track is suspended by the system, we also mute it if it is not already muted.
+    // As it is possible that the track could still flow data even if it is system muted
+    const handleSuspended = (track) => {
+        switch (track.kind) {
+            case 'audioinput': {
+                if (props.audioAlreadyMuted === undefined) {
+                    props.audioAlreadyMuted = props.audioTrack?.muted === true;
+                }
+                if (!props.audioAlreadyMuted) {
+                    props.audioTrack?.mute(true, true);
+                }
+                break;
+            }
+            case 'videoinput': {
+                if (props.videoAlreadyMuted === undefined) {
+                    props.videoAlreadyMuted = props.videoTrack?.muted === true;
+                }
+                if (!props.videoAlreadyMuted) {
+                    props.videoTrack?.mute(true, true);
+                }
+                break;
+            }
+        }
+    };
+    // When the track is resumed by the system, we only unmute the track
+    // if an only if it is muted by the suspended event
+    const handleResumed = (track) => {
+        switch (track.kind) {
+            case 'audioinput': {
+                assert(props.audioAlreadyMuted !== undefined, 'Audio Resume should not be triggerred before Suspended');
+                if (props.audioTrack?.suspended === false) {
+                    if (!props.audioAlreadyMuted) {
+                        props.audioTrack.mute(false);
+                    }
+                    props.audioAlreadyMuted = undefined;
+                }
+                break;
+            }
+            case 'videoinput': {
+                assert(props.videoAlreadyMuted !== undefined, 'Video Resume should not be triggerred before Suspended');
+                if (props.videoTrack?.suspended === false) {
+                    if (!props.videoAlreadyMuted) {
+                        props.videoTrack.mute(false);
+                    }
+                    props.videoAlreadyMuted = undefined;
+                }
+                break;
+            }
         }
-        if (previousMuteState !== currentMuteState) {
-            mediaInit.signals?.[track.kind === 'audioinput'
-                ? 'onAudioMuteStateChanged'
-                : 'onVideoMuteStateChanged']?.emit(currentMuteState);
+    };
+    const subscribe = () => [
+        onDevicesChanged.add(updateDevices),
+        mediaInit.signals?.onMediaTrackSuspended?.add(handleSuspended),
+        mediaInit.signals?.onMediaTrackResumed?.add(handleResumed),
+    ];
+    let subscriptions = subscribe();
+    const unsubscribe = () => {
+        for (const unsubscribe of subscriptions) {
+            unsubscribe?.();
         }
+        subscriptions.length = 0;
     };
     return {
         get id() {
@@ -372,16 +409,16 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
                     props.devices.size('videoinput') > 0));
         },
         muteAudio(mute) {
-            if (!props.audioTrack || props.audioTrack.muted === mute) {
+            if (!props.audioTrack || props.audioTrack.suspended) {
                 return;
             }
-            muteTrack(props.audioTrack, mute);
+            props.audioTrack.mute(mute);
         },
         muteVideo(mute) {
-            if (!props.videoTrack || props.videoTrack.muted === mute) {
+            if (!props.videoTrack || props.videoTrack.suspended) {
                 return;
             }
-            muteTrack(props.videoTrack, mute);
+            props.videoTrack?.mute(mute);
         },
         applyConstraints: async (constraints) => {
             await Promise.all([
@@ -394,12 +431,7 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             ]);
         },
         async release() {
-            unsubscribe?.();
-            unsubscribe = undefined;
-            for (const unsubscribe of trackSubscriptions.values()) {
-                unsubscribe();
-            }
-            trackSubscriptions.clear();
+            unsubscribe();
             await Promise.all([
                 props.audioTrack?.release(),
                 props.videoTrack?.release(),
@@ -409,13 +441,13 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             audio: props.audioTrack?.getSettings(),
             video: props.videoTrack?.getSettings(),
         }),
-        clone(signals) {
+        clone(signals, label) {
             const tracks = [props.audioTrack, props.videoTrack].flatMap(track => {
                 if (!track?.source) {
                     return [];
                 }
-                const cloned = track.source.clone(signals);
-                return cloned;
+                const cloned = track.source.clone(signals, label);
+                return [cloned];
             });
             const stream = this.stream &&
                 new MediaStream(tracks.flatMap(track => (track.track ? [track.track] : [])));
@@ -440,6 +472,9 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
         getVideoTracks() {
             return props.videoTrack ? [props.videoTrack] : [];
         },
+        isCurrentTrack(track) {
+            return track === props.audioTrack || track === props.videoTrack;
+        },
         addTrack(track) {
             switch (track.kind) {
                 case 'audioinput':
@@ -457,13 +492,22 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
                 default:
                     break;
             }
+            if (subscriptions.length === 0) {
+                subscriptions = subscribe();
+            }
             if (track.track) {
                 props.stream.addTrack(track.track);
-                if (!trackSubscriptions.has(track.track)) {
-                    trackSubscriptions.set(track.track, subscribeTrackAndSourceTrackEvents(track));
-                }
                 mediaInit.signals?.onAddTrack?.emit(track.track);
                 props.stream.dispatchEvent(new MediaStreamTrackEvent('addtrack', { track: track.track }));
+                if (track.muted) {
+                    mediaInit.signals?.onMediaTrackMuted?.emit(track);
+                }
+                if (track.suspended) {
+                    mediaInit.signals?.onMediaTrackSuspended?.emit(track);
+                }
+                if (track.stopped) {
+                    mediaInit.signals?.onMediaTrackStopped?.emit(track);
+                }
             }
         },
         removeTrack(track) {
@@ -478,13 +522,14 @@ export const buildMedia = (mediaInit, onDevicesChanged = internalSignals.onDevic
             }
             if (removed && track.track) {
                 props.stream.removeTrack(track.track);
-                trackSubscriptions.get(track.track)?.();
-                trackSubscriptions.delete(track.track);
                 mediaInit.signals?.onRemoveTrack?.emit(track.track);
                 props.stream.dispatchEvent(new MediaStreamTrackEvent('removetrack', {
                     track: track.track,
                 }));
             }
+            if (subscriptions.length) {
+                unsubscribe();
+            }
         },
         toJSON() {
             return {

package/dist/videoProcessor.d.ts

@@ -1,10 +1,10 @@
-import type { VideoProcessor, SegmentationTransform, SegmentationModel, RenderBackend } from '@pexip/media-processor';
+import type { RenderBackend, SegmentationModel, SegmentationTransform, VideoProcessor } from '@pexip/media-processor';
 import type { MediaDeviceRequest } from '@pexip/media-control';
-import type { TrackProcessor, VideoRenderParams, Segmenters, VideoStreamTrackProcessorAPIs, VideoContentHint } from './types';
+import type { MediaSignals, Segmenters, TrackProcessor, VideoContentHint, VideoRenderParams, VideoStreamTrackProcessorAPIs } from './types';
 interface ProcessorDeps {
-    videoProcessor?: () => VideoProcessor;
-    transformer?: SegmentationTransform;
     segmenters: Partial<Segmenters>;
+    transformer?: SegmentationTransform;
+    videoProcessor?: () => VideoProcessor;
     videoSegmentationModel?: SegmentationModel;
 }
 interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<ProcessorDeps, 'videoProcessor'> {
@@ -28,6 +28,7 @@ interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<Pro
     stopAsMute?: () => boolean;
     dynamicProcessingDimensions?: () => boolean;
     gpuAPI?: () => RenderBackend;
+    signals?: MediaSignals;
 }
 interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<ProcessorDeps> {
     hasInitialized: boolean;
@@ -37,5 +38,5 @@ declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUr
 type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
 type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest["video"], props: FeatureProps) => FeatureProps;
-export declare const createVideoStreamProcess: ({ backgroundImageUrl, dynamicProcessingDimensions, edgeBlurAmount, foregroundThreshold, frameRate, gpuAPI, label, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute, trackProcessorAPI, videoSegmentation, ...options }: VideoStreamProcessOptions) => TrackProcessor;
+export declare const createVideoStreamProcess: ({ backgroundImageUrl, dynamicProcessingDimensions, edgeBlurAmount, foregroundThreshold, frameRate, gpuAPI, label, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute, trackProcessorAPI, videoSegmentation, signals, ...options }: VideoStreamProcessOptions) => TrackProcessor;
 export {};

package/dist/videoProcessor.js

@@ -1,5 +1,5 @@
-import { createVideoProcessor, createCanvasTransform, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
-import { muteStreamTrack, extractConstraintsWithKeys, getValueFromConstrainNumber, } from '@pexip/media-control';
+import { createCanvasTransform, createVideoProcessor, createVideoTrackProcessor, createVideoTrackProcessorWithFallback, isRenderEffects, isSegmentationModel, } from '@pexip/media-processor';
+import { extractConstraintsWithKeys, getValueFromConstrainNumber, muteStreamTrack, } from '@pexip/media-control';
 import { isEmpty, assert } from '@pexip/utils';
 import { createMediaTrack, getBlurKernelSize } from './utils';
 import { logger, proxyWithLog } from './logger';
@@ -142,7 +142,7 @@ const getTrackProcessor = (shouldUseStreamTrackProcessor, ...params) => {
     return createVideoTrackProcessorWithFallback(...params);
 };
 export const createVideoStreamProcess = ({ backgroundImageUrl, dynamicProcessingDimensions = () => false, edgeBlurAmount, foregroundThreshold, frameRate, gpuAPI = () => 'webgl', // dynamically changing the config will not be picked up, not a priority for now
-label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute = () => false, trackProcessorAPI = () => 'stream', videoSegmentation, ...options }) => {
+label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute = () => false, trackProcessorAPI = () => 'stream', videoSegmentation, signals, ...options }) => {
     const videoSegmentationModel = options.videoSegmentationModel ?? 'selfie';
     const segmenter = options.segmenters[videoSegmentationModel];
     if (!segmenter) {
@@ -253,36 +253,61 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
         assert(track, 'Video track should be there');
         // Inherit contentHint from previous track
         track.contentHint = prevMediaTrack.track.contentHint;
-        const release = async () => {
-            props.videoProcessor().close();
-            await prevMediaTrack.release();
-            _videoProcessor = undefined;
-            props.hasInitialized = false;
-        };
-        const mute = (mute) => {
-            props.transformer.effects = mute
-                ? 'none'
-                : props.videoSegmentation ?? 'none';
-            muteStreamTrack(stream)(mute, 'video');
-            if (mute && stopAsMute()) {
-                for (const track of stream.getVideoTracks()) {
-                    track.stop();
-                }
-                release();
-            }
-            prevMediaTrack.mute(mute);
-        };
+        let trackReleaseTimeoutID = 0;
         return createMediaTrack({
             label,
             kind: 'videoinput',
-            constraints: prevMediaTrack.getConstraints(),
-            previousMediaTrack: prevMediaTrack,
-            input: prevMediaTrack.input,
-            expectedInput: prevMediaTrack.expectedInput,
+            get constraints() {
+                return this.previousMediaTrack?.getConstraints();
+            },
+            get previousMediaTrack() {
+                return prevMediaTrack;
+            },
+            get input() {
+                return this.previousMediaTrack?.input;
+            },
+            get expectedInput() {
+                return this.previousMediaTrack?.expectedInput;
+            },
             track,
-            mute,
-            release,
-            applyConstraints: async (constraints) => {
+            overrideMute: true,
+            get muted() {
+                return this.previousMediaTrack?.muted;
+            },
+            mute(mute, _self, soft) {
+                props.transformer.effects = mute
+                    ? 'none'
+                    : props.videoSegmentation ?? 'none';
+                muteStreamTrack(stream)(mute, 'video');
+                if (stopAsMute() && !soft) {
+                    if (mute) {
+                        // Track cannot be stopped immediately as we need to ensure
+                        // the last frame is black to avoid frozen frame in a gateway call
+                        trackReleaseTimeoutID = window.setTimeout(() => {
+                            for (const track of stream.getVideoTracks()) {
+                                track.stop();
+                            }
+                            void this?.release?.();
+                            trackReleaseTimeoutID = 0;
+                        }, 500);
+                    }
+                    else {
+                        if (trackReleaseTimeoutID) {
+                            window.clearTimeout(trackReleaseTimeoutID);
+                            trackReleaseTimeoutID = 0;
+                        }
+                    }
+                }
+                this.previousMediaTrack?.mute(mute, soft);
+            },
+            signals,
+            async release() {
+                props.videoProcessor().close();
+                await this.previousMediaTrack?.release();
+                _videoProcessor = undefined;
+                props.hasInitialized = false;
+            },
+            async applyConstraints(constraints) {
                 if (isEmpty(constraints)) {
                     return;
                 }
@@ -293,8 +318,8 @@ label = PROCESSOR_LABELS.VideoProcessor, lowestProcessingHeight, maskCombineRati
                 }
                 applyFeatures(props.transformer, features, lowestProcessingHeight);
                 if (dynamicProcessingDimensions()) {
-                    const acceptedSettings = prevMediaTrack.source.getSettings();
-                    if (acceptedSettings.width && acceptedSettings.height) {
+                    const acceptedSettings = this.previousMediaTrack?.source.getSettings();
+                    if (acceptedSettings?.width && acceptedSettings?.height) {
                         props.transformer.update({
                             processingHeight: acceptedSettings.height,
                             processingWidth: acceptedSettings.width,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "20.2.0",
+  "version": "20.3.0",
   "description": "Home for media related stuff",
   "homepage": "https://gitlab.com/pexip/zoo",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -44,14 +44,14 @@
     "typecheck": "yarn tsc --noEmit -p ."
   },
   "dependencies": {
-    "@pexip/media-control": "20.2.0",
-    "@pexip/media-processor": "20.2.0",
+    "@pexip/media-control": "20.3.0",
+    "@pexip/media-processor": "20.3.0",
     "@pexip/signal": "16.9.0",
-    "@pexip/utils": "17.0.0"
+    "@pexip/utils": "17.0.1"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@pexip/bundler": "18.1.1",
+    "@pexip/bundler": "18.1.2",
     "@swc/core": "^1.10.12",
     "@swc/jest": "^0.2.37",
     "jest": "^29.5.12",