@pexip/media 18.4.0 → 19.0.0

package/CHANGELOG.md

@@ -1,5 +1,135 @@
 # @pexip/media
 
+## 19.0.0
+
+### Major Changes
+
+- 878fcca: Decouple Audio/Video request
+
+  The library is no longer assuming the request constraints both video and
+  audio, and there is no retry mechanism builtin. The media request and
+  processing are decoupled as well to minimize the time to get the media
+  flowing.
+
+  Changes of `Media` object:
+
+  - `MediaTrack`-based construct: An `Media` construct is changed to base on
+    `MediaTrack` which is a wrapper to `MediaStreamTrack` with a linked list to
+    reference on different `MediaStreamTrack` produced by the processor as well
+    as the original source track obtained from `getUserMedia` API.
+  - All track related functions are implemented by `MediaTrack`, e.g.
+    `applyConstraints`, `getSettings` and `clone`, etc.
+  - `Media['getSettings']` returns
+    `{audio?: ExtendedMediaTrackSettings, video?: ExtendedMediaTrackSettings}`
+  - The following attributes/functions are added to media object:
+    - `active: boolean`;
+    - `id: string`;
+    - `getConstraints(): MediaDeviceRequest`;
+    - `setOriginalConstraints(constraints: MediaDeviceRequest): void`;
+    - `getOriginalConstraints(): MediaDeviceRequest`;
+    - `isAudioInputUnavailable(): boolean`;
+    - `isVideoInputUnavailable(): boolean`;
+    - `requestedAudio(): boolean`;
+    - `requestedVideo(): boolean`;
+    - `getTracks(): MediaTrack[]`;
+    - `getAudioTracks(): MediaTrack[]`;
+    - `getVideoTracks(): MediaTrack[]`;
+    - `addTrack(track: MediaTrack): void`;
+    - `removeTrack(track: MediaTrack): void`;
+    - `clone(): Media`;
+  - BREAKING CHANGE:
+    - `Media['rawStream']` is removed, and we no longer keep a reference of the
+      original `MediaStream`, we keep the source track instead. See
+      `MediaTrack['source']`.
+    - `Media['constraints']` is removed, please use `Media['getConstraints']` or
+      `Media['getOriginalConstraints']` to get the constraints.
+    - The `processors` parameter for `createMedia({processors, ...})` has been
+      split into `audioProcessors` and `videoProcessors`, i.e.
+      `createMedia({videoProcessors, audioProcessors, ...})`
+
+  Changes of processing pipeline:
+
+  - The pipeline is changed to based on `MediaTrack`.
+  - Video processing pipeline and Audio processing pipeline are run in parallel
+    whereas the pipeline itself is run in series
+  - `getUserMedia` and processing pipeline are decoupled
+  - `MediaStream` is created as soon as the `Media` object is created which
+    could contains no track.
+  - Any subsequent `getUserMedia` API call will update the track in the
+    `MediaStream` object. i.e. `MediaStream` will remain the same through out
+    the whole life cycle of the App and the tracks are updated according the
+    `getUserMedia` with an assumption that a `MediaStream` contains at most 1
+    audio track and 1 video track. `addtrack` and `removetrack` events are
+    triggered whenever there is any changes to the `MediaStream`
+
+  Changes of media signals;
+
+  The following signals are added
+
+  - `onAddTrack`: Emit when a new track is added to the `MediaStream`
+  - `onRemoveTrack`: Emit when a track from the `MediaStream` is removed
+  - `onUpdatingMedia`: Emit whenever a track is added or removed from the
+    `Media` object
+  - `onTrackReleased`: Emit when a track is stopped
+
+  The behavior of the following signals are changed:
+
+  - `onAudioMuteStateChanged`: Emit whenever the mute state of the `MediaTrack`
+    is changed or the mute state of the source track. Here the mute state
+    includes the following: - `MediaStreamTrack.enabled` -
+    `MediaStreamTrack.muted` triggered by `mute` and `unmute` event -
+    `readyState` is `ended` triggered by `ended` event
+  - `onVideoMuteStateChanged`: Emit whenever the mute state of the `MediaTrack`
+    is changed or the mute state of the source track. Here the mute state
+    includes the following: - `MediaStreamTrack.enabled` -
+    `MediaStreamTrack.muted` triggered by `mute` and `unmute` event -
+    `readyState` is `ended` triggered by `ended` event
+
+### Minor Changes
+
+- 029198f: Fix device not found error mappinng
+- b13010c: Handle timeout error
+- 7d9eac2: Add processed track state signals
+
+  - onStreamTrackEndedFinal - Emitted when the final track is ended
+  - onStreamTrackMutedFinal - Emitted when the final track is muted
+  - onStreamTrackUnmutedFinal - Emitted when the final track is unmuted
+
+### Patch Changes
+
+- Updated dependencies [6c42e61]
+- Updated dependencies [e362a65]
+- Updated dependencies [22a807b]
+- Updated dependencies [878fcca]
+- Updated dependencies [5b9daa4]
+- Updated dependencies [164a3e2]
+- Updated dependencies [b13010c]
+- Updated dependencies [66a8f72]
+- Updated dependencies [4fd97b2]
+- Updated dependencies [db258ca]
+- Updated dependencies [dfd1900]
+  - @pexip/media-control@19.0.0
+  - @pexip/media-processor@19.0.0
+  - @pexip/utils@16.13.0
+  - @pexip/signal@16.7.1
+
+## 18.5.0
+
+### Minor Changes
+
+- f146682: Make `delegate` prop to be a fn
+
+### Patch Changes
+
+- d98fe9d: Move to module type.
+- Updated dependencies [d98fe9d]
+- Updated dependencies [acd7aba]
+- Updated dependencies [f146682]
+  - @pexip/media-processor@18.5.0
+  - @pexip/media-control@18.5.0
+  - @pexip/signal@16.7.1
+  - @pexip/utils@16.12.1
+
 ## 18.4.0
 
 ### Patch Changes

package/README.md

@@ -19,10 +19,11 @@ Its major features:
 - Create a media pipeline to get and process the `MediaStream`, see
   `createMediaPipeline`. It also provides some media features including:
 
-      - Video Processing - background blur
-      - Audio Processing - Noise Suppression
-      - Audio Signal Detection - To detect if the active audio input device is capturing any audio
-      - Voice Activity Detection - To detect if there is any voice activity
+  - Video Processing - background blur
+  - Audio Processing - Noise Suppression
+  - Audio Signal Detection - To detect if the active audio input device is
+    capturing any audio
+  - Voice Activity Detection - To detect if there is any voice activity
 
 - Provide media input related information
 - Verify if we need to request a new `MediaStream`, see `updateMedia`
@@ -40,7 +41,7 @@ A processor to process video from the stream to be used together with
 
 ### `createPreviewController`
 
-It create's an object to control the provided media stream, e.g. change the
+It creates an object to control the provided media stream, e.g. change the
 audio/video input device and apply changes to the main stream when necessary.
 
 ## Usage
@@ -195,10 +196,9 @@ const transformer = createCanvasTransform(segmenter, {
   backgroundImageUrl: states.backgroundImageUrl,
 });
 
-const delegate = isChromium() ? 'GPU' : 'CPU';
 const selfie = createSegmenter(taskVisionBasePath, {
   modelAsset,
-  delegate,
+  delegate: () => 'GPU', // Use GPU
 });
 
 export const segmenters: Partial<Segmenters> = {
@@ -218,7 +218,8 @@ const videoStreamProcessor = createVideoStreamProcess({
 export const mediaController = createMedia({
   getMuteState: () => states.muted,
   signals,
-  mediaProcessors: [videoStreamProcessor],
+  audioProcessors: [], // No audio processor
+  videoProcessors: [videoStreamProcessor],
 });
 
 // Hook-up the signals to get the update

package/dist/audioMixingProcessor.d.ts

@@ -1,6 +1,6 @@
 import type { MediaDeviceRequest } from '@pexip/media-control';
 import type { AudioGraph, AudioNodeInit } from '@pexip/media-processor';
-import type { Process, Media } from './types';
+import type { TrackProcessor } from './types';
 interface AudioStreamProcessorProps {
     mixWithAdditionalMedia?: boolean;
     merger?: AudioNodeInit<ChannelMergerNode, ChannelMergerNode>;
@@ -14,5 +14,5 @@ export declare const updateFeatureProps: (constraints: MediaDeviceRequest['audio
 /**
  * Create a Audio Mixing Processor and will own the stream passed-in
  */
-export declare const createAudioMixingProcess: (getCurrrentMedia: () => MediaStream | undefined, scope?: string) => Process<Promise<Media>>;
+export declare const createAudioMixingProcess: (getCurrentMedia: () => MediaStream | undefined, label?: "AudioMixingProcessor") => TrackProcessor;
 export {};

package/dist/audioMixingProcessor.js

@@ -1,8 +1,9 @@
-import { muteStreamTrack, stopMediaStream, extractConstraintsWithKeys, } from '@pexip/media-control';
-import { isEmpty } from '@pexip/utils';
+import { extractConstraintsWithKeys } from '@pexip/media-control';
+import { isEmpty, assert } from '@pexip/utils';
 import { createAudioGraph, createAudioGraphProxy, createChannelMergerGraphNode, createStreamDestinationGraphNode, createStreamSourceGraphNode, resumeAudioOnUnmute, } from '@pexip/media-processor';
 import { logger } from './logger';
-import { applyExtendedConstraints, shallowCopy, wrapToJSON } from './utils';
+import { createMediaTrack, isTrackMuted } from './utils';
+import { PROCESSOR_LABELS } from './constants';
 const FEATURE_KEYS = ['mixWithAdditionalMedia'];
 const getAudioConstraints = extractConstraintsWithKeys(FEATURE_KEYS);
 export const updateFeatureProps = (constraints, props) => {
@@ -19,35 +20,31 @@ export const updateFeatureProps = (constraints, props) => {
 /**
  * Create a Audio Mixing Processor and will own the stream passed-in
  */
-export const createAudioMixingProcess = (getCurrrentMedia, scope = 'mixer') => {
+export const createAudioMixingProcess = (getCurrentMedia, label = PROCESSOR_LABELS.AudioMixingProcessor) => {
     const props = {};
-    return async (mediaP) => {
-        const media = await mediaP;
-        updateFeatureProps(media.constraints?.audio, props);
-        const [mainTrack] = media.stream?.getAudioTracks() ?? [];
-        const displayStream = getCurrrentMedia();
-        const [displayTrack] = displayStream?.getAudioTracks?.() ?? [];
-        const applyConstraints = applyExtendedConstraints(media, async (constraints) => {
-            if (isEmpty(constraints.audio)) {
+    return async (prevMediaTrack) => {
+        updateFeatureProps(prevMediaTrack.getConstraints(), props);
+        const displayStream = getCurrentMedia();
+        const displayTrack = displayStream?.getAudioTracks?.().at(0);
+        const applyConstraints = async (constraints) => {
+            if (isEmpty(constraints)) {
                 return;
             }
-            const features = updateFeatureProps(constraints.audio, props);
-            logger.debug({ scope, constraints: constraints.audio, features }, 'apply audio mixing constraints');
+            const features = updateFeatureProps(constraints, props);
+            logger.debug({ label, constraints: constraints, features }, 'apply audio mixing constraints');
             if (isEmpty(features) ||
                 (props.audioGraph &&
                     ['closed', 'closing'].includes(props.audioGraph.state))) {
                 return;
             }
             if (features.mixWithAdditionalMedia) {
-                const newStream = getCurrrentMedia();
-                const [newTrack] = getCurrrentMedia()?.getAudioTracks() ?? [];
+                const newStream = getCurrentMedia();
+                const [newTrack] = getCurrentMedia()?.getAudioTracks() ?? [];
                 if (!newTrack) {
                     return;
                 }
                 if (!props.audioGraph || !props.merger) {
-                    // TODO: Should update the media stream directly
-                    // return replace(media.stream, newTrack);
-                    logger.debug({ scope }, 'Should update the media stream directly');
+                    logger.debug({ label }, 'Should update the media stream directly');
                     return;
                 }
                 if (props.displaySource && props.merger) {
@@ -78,96 +75,112 @@ export const createAudioMixingProcess = (getCurrrentMedia, scope = 'mixer') => {
                 }
             }
             return Promise.resolve();
-        });
-        if (!media.stream || (!mainTrack && !displayTrack)) {
-            return media;
-        }
-        if (!mainTrack && displayTrack) {
-            media.stream?.addTrack(displayTrack);
-            return shallowCopy(media, {
-                muteAudio: () => {
+        };
+        // No need to mix and use as is
+        if (!prevMediaTrack.track) {
+            if (!displayTrack) {
+                // Do nothing
+                return prevMediaTrack;
+            }
+            return createMediaTrack({
+                label,
+                kind: 'audioinput',
+                previousMediaTrack: prevMediaTrack,
+                input: prevMediaTrack.input,
+                expectedInput: prevMediaTrack.expectedInput,
+                track: displayTrack,
+                constraints: prevMediaTrack.getConstraints(),
+                overrideMute: true,
+                mute: () => {
                     /* Do Nothing */
                 },
                 applyConstraints,
-            });
-        }
-        try {
-            const mainSource = createStreamSourceGraphNode(media.stream);
-            props.displaySource =
-                displayStream && createStreamSourceGraphNode(displayStream);
-            props.merger = createChannelMergerGraphNode();
-            const destination = createStreamDestinationGraphNode({
-                channelCount: 1,
-                channelCountMode: 'explicit',
-            });
-            const initialAudioNodeConnection = [
-                [mainSource, props.merger],
-                [props.merger, destination],
-            ];
-            logger.debug({ initialAudioNodeConnection, scope }, 'Initial AudioNodeConnection');
-            const audioGraph = createAudioGraphProxy(createAudioGraph(initialAudioNodeConnection), {
-                connect: (target, args) => {
-                    logger.debug({ scope, target, args }, 'connect nodes');
-                },
-                disconnect: (target, args) => {
-                    logger.debug({ scope, target, args }, 'disconnect nodes');
-                },
-            });
-            props.audioGraph = audioGraph;
-            const unsubscribes = media.rawStream
-                ?.getAudioTracks()
-                .map(resumeAudioOnUnmute(audioGraph.context));
-            if (props.displaySource) {
-                props.audioGraph.connect([props.displaySource, props.merger]);
-            }
-            const tracks = [
-                ...(destination?.node?.stream.getAudioTracks() ?? []),
-                ...(media.stream?.getVideoTracks() ?? []),
-            ];
-            const stream = new MediaStream(tracks);
-            const release = async () => {
-                logger.debug({ scope }, 'Release Media');
-                unsubscribes?.forEach(unsubscribe => unsubscribe());
-                stopMediaStream(stream);
-                // Release Props
-                await props.audioGraph?.release();
-                await media.release();
-                props.audioGraph = undefined;
-                props.merger = undefined;
-                props.displaySource = undefined;
-            };
-            const muteAudio = (mute) => {
-                media.muteAudio(mute);
-                // TODO: dbl check if this is safe
-                muteStreamTrack(mainSource.audioNode?.mediaStream)(mute, 'audio');
-            };
-            const prevGetSettings = media.getSettings;
-            return wrapToJSON(shallowCopy(media, {
-                stream,
-                applyConstraints,
-                muteAudio,
-                release,
                 getSettings: () => {
-                    const { audio, video } = prevGetSettings();
                     const mixWithAdditionalMedia = !!props.mixWithAdditionalMedia;
-                    const audioSettings = {
+                    const contentHint = displayTrack?.contentHint ?? '';
+                    return {
                         mixWithAdditionalMedia,
+                        contentHint,
                     };
-                    const settings = {
-                        audio: audio.map(settings => ({
-                            ...settings,
-                            ...audioSettings,
-                        })),
-                        video,
-                    };
-                    logger.debug({ scope, settings: audioSettings }, 'get audio mixing processor settings');
-                    return settings;
                 },
-            }));
+            });
         }
-        catch (error) {
-            logger.error({ scope, error }, 'Unable to use WebAudio, return the raw media instead');
-            return media;
+        assert(prevMediaTrack.track, 'Main track should be available');
+        const mainSource = createStreamSourceGraphNode(new MediaStream([prevMediaTrack.track]));
+        props.displaySource =
+            displayStream && createStreamSourceGraphNode(displayStream);
+        props.merger = createChannelMergerGraphNode();
+        const destination = createStreamDestinationGraphNode({
+            channelCount: 1,
+            channelCountMode: 'explicit',
+        });
+        const initialAudioNodeConnection = [
+            [mainSource, props.merger],
+            [props.merger, destination],
+        ];
+        logger.debug({ initialAudioNodeConnection, label }, 'Initial AudioNodeConnection');
+        const audioGraph = createAudioGraphProxy(createAudioGraph(initialAudioNodeConnection), {
+            connect: (target, args) => {
+                logger.debug({ label, target, args }, 'connect nodes');
+            },
+            disconnect: (target, args) => {
+                logger.debug({ label, target, args }, 'disconnect nodes');
+            },
+        });
+        props.audioGraph = audioGraph;
+        const unsubscribe = resumeAudioOnUnmute(audioGraph.context)(prevMediaTrack.track);
+        if (props.displaySource) {
+            props.audioGraph.connect([props.displaySource, props.merger]);
         }
+        const track = destination?.node?.stream.getAudioTracks().at(0);
+        assert(track, 'No audio track mixed');
+        // Inherit contentHint from previous track
+        track.contentHint = prevMediaTrack.track.contentHint;
+        const release = async () => {
+            logger.debug({ label }, 'Release Media');
+            unsubscribe();
+            // Release Props
+            await props.audioGraph?.release();
+            props.audioGraph = undefined;
+            props.merger = undefined;
+            props.displaySource = undefined;
+        };
+        return Promise.resolve(createMediaTrack({
+            label,
+            kind: 'audioinput',
+            previousMediaTrack: prevMediaTrack,
+            input: prevMediaTrack.input,
+            expectedInput: prevMediaTrack.expectedInput,
+            track,
+            overrideMute: true,
+            mute: toMute => {
+                prevMediaTrack.mute(toMute);
+                mainSource.node?.mediaStream
+                    .getAudioTracks()
+                    .forEach(track => {
+                    track.enabled = !toMute;
+                });
+            },
+            get muted() {
+                return (
+                // If the source track muted, the following processed track will
+                // be muted as well since there is no data to process
+                !!prevMediaTrack.muted ||
+                    mainSource.node?.mediaStream
+                        .getAudioTracks()
+                        .some(isTrackMuted));
+            },
+            constraints: prevMediaTrack.getConstraints(),
+            applyConstraints,
+            release,
+            getSettings: () => {
+                const mixWithAdditionalMedia = !!props.mixWithAdditionalMedia;
+                const contentHint = track.contentHint ?? '';
+                return {
+                    mixWithAdditionalMedia,
+                    contentHint,
+                };
+            },
+        }));
     };
 };

package/dist/audioProcessor.d.ts

@@ -1,6 +1,6 @@
 import type { AudioGraphOptions, AudioNodeInit, ThrottleOptions, DenoiseWorkletNodeInit, AnalyzerNodeInit, AudioGraph } from '@pexip/media-processor';
 import type { MediaDeviceRequest } from '@pexip/media-control';
-import type { Process, Media, DenoiseParams, AudioContentHint } from './types';
+import type { TrackProcessor, DenoiseParams, AudioContentHint } from './types';
 type AudioNodeInits = AudioNodeInit[];
 /**
  * A function to be called to create the AudioNodes needed for the graph
@@ -8,7 +8,7 @@ type AudioNodeInits = AudioNodeInit[];
  *
  * @param media - Media to be used for the AudioGraph creation
  */
-type CreateNodes = (media: Media) => AudioNodeInits;
+type CreateNodes = (track: MediaStreamTrack) => AudioNodeInits;
 interface AudioProcessOptions {
     /**
      * An option is being passed to AnalyserNode creation when used
@@ -59,7 +59,7 @@ interface AudioProcessOptions {
      * silent in FFTed time domain
      */
     silentThreshold?: number;
-    scope?: string;
+    label?: string;
 }
 interface AudioStreamProcessorProps {
     audioGraphOptions?: AudioGraphOptions;
@@ -82,5 +82,5 @@ export declare const updateFeatureProps: (constraints: MediaDeviceRequest['audio
 /**
  * Create a Audio Stream Processor and will own the stream passed-in
  */
-export declare const createAudioStreamProcess: ({ analyzerUpdateFrequency, audioGraphOptions, audioSignalDetectionDuration, clock, createNodes, denoiseParams, fftSize, onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold, throttleMs, scope, }: AudioProcessOptions & ThrottleOptions) => Process<Promise<Media>>;
+export declare const createAudioStreamProcess: ({ analyzerUpdateFrequency, audioGraphOptions, audioSignalDetectionDuration, clock, createNodes, denoiseParams, fftSize, onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold, throttleMs, label, }: AudioProcessOptions & ThrottleOptions) => TrackProcessor;
 export {};