@pexip/media 18.4.0 → 19.0.0

package/dist/audioProcessor.js

@@ -1,9 +1,10 @@
-import { stopMediaStream, extractConstraintsWithKeys, muteStreamTrack, } from '@pexip/media-control';
-import { createQueue, isEmpty } from '@pexip/utils';
+import { extractConstraintsWithKeys } from '@pexip/media-control';
+import { createQueue, isEmpty, assert } from '@pexip/utils';
 import { createAudioGraph, createAudioGraphProxy, createStreamSourceGraphNode, createStreamDestinationGraphNode, createAnalyzerSubscribableGraphNode, createDenoiseWorkletGraphNode, createAudioSignalDetector, createVADetector, createVoiceDetectorFromTimeData, createVoiceDetectorFromProbability, avg, } from '@pexip/media-processor';
+import { PROCESSOR_LABELS } from './constants';
 import { logger } from './logger';
 import { isAudioContentHint } from './typeGuard';
-import { shallowCopy, wrapToJSON, applyExtendedConstraints } from './utils';
+import { createMediaTrack } from './utils';
 /**
  * Fetch the wasm when it doesn't exist from the provided, otherwise do nothing
  *
@@ -58,7 +59,7 @@ audioGraphOptions, audioSignalDetectionDuration = 4.0, // 4 seconds
 clock, createNodes, denoiseParams, fftSize = 2048, // FFT size
 onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold = 10.0 / 32767, // At least one LSB 16-bit data (compare is on absolute value).
 throttleMs = 3000, // 3 seconds
-scope = 'media', }) => {
+label = PROCESSOR_LABELS.AudioProcessor, }) => {
     const props = {
         audioGraphOptions,
         vad: false,
@@ -86,7 +87,7 @@ scope = 'media', }) => {
             }
             catch (error) {
                 logger.error({
-                    scope,
+                    label,
                     error,
                     moduleURL: denoiseParams?.workletModule,
                     options: denoiseParams?.workletOptions,
@@ -99,7 +100,7 @@ scope = 'media', }) => {
         }
         catch (error) {
             logger.error({
-                scope,
+                label,
                 error,
                 url: denoiseParams?.wasmURL,
                 prevWasm: props.denoiseWasm,
@@ -144,167 +145,131 @@ scope = 'media', }) => {
         props.analyzer = analyzer;
         return analyzer;
     };
-    return async (mediaP) => {
-        const media = await mediaP;
-        updateFeatureProps(media.constraints?.audio, props);
+    return async (prevMediaTrack) => {
+        updateFeatureProps(prevMediaTrack.getConstraints(), props);
         const shouldProcessAudio = shouldEnable() &&
-            !!media.stream?.getAudioTracks().length &&
             (!!onVoiceActivityDetected ||
                 !!onAudioSignalDetected ||
-                props.asd ||
-                props.vad ||
-                props.denoise ||
+                !!props.asd ||
+                !!props.vad ||
+                !!props.denoise ||
                 !!createNodes);
-        if (!media.stream?.getAudioTracks().length || !shouldProcessAudio) {
-            return media;
+        if (!shouldProcessAudio || !prevMediaTrack.track) {
+            return prevMediaTrack;
         }
-        try {
-            const source = createStreamSourceGraphNode(media.stream);
-            const destination = createStreamDestinationGraphNode();
-            const otherNodes = createNodes?.(media) ?? [];
-            const analyzer = createAnalyzer();
-            const initialAudioNodeConnection = [
-                [source, ...otherNodes, destination],
-                [source, analyzer],
-            ];
-            logger.debug({ initialAudioNodeConnection, scope }, 'Initial AudioNodeConnection');
-            const audioGraph = createAudioGraphProxy(createAudioGraph(initialAudioNodeConnection, props.audioGraphOptions), {
-                connect: (target, args) => {
-                    logger.debug({ scope, target, args }, 'connect nodes');
-                },
-                disconnect: (target, args) => {
-                    logger.debug({ scope, target, args }, 'disconnect nodes');
-                },
-            });
-            props.audioGraph = audioGraph;
-            const denoiseNode = await createDenoiseNode(props.denoise);
-            const connectDenoise = (node) => {
-                if (node) {
-                    audioGraph.disconnect([source, ...otherNodes, destination]);
-                    audioGraph.connect([
-                        source,
-                        node,
-                        ...otherNodes,
-                        destination,
-                    ]);
-                }
-            };
-            connectDenoise(denoiseNode);
-            const tracks = [
-                ...(destination?.node?.stream.getAudioTracks() ?? []),
-                ...(media.stream?.getVideoTracks().map(t => t.clone()) ?? []),
-            ];
-            const stream = new MediaStream(tracks);
-            const applyConstraints = applyExtendedConstraints(media, async (constraints) => {
-                if (isEmpty(constraints.audio)) {
+        const source = createStreamSourceGraphNode(new MediaStream([prevMediaTrack.track]));
+        const destination = createStreamDestinationGraphNode();
+        const otherNodes = createNodes?.(prevMediaTrack.track) ?? [];
+        const analyzer = createAnalyzer();
+        const initialAudioNodeConnection = [
+            [source, ...otherNodes, destination],
+            [source, analyzer],
+        ];
+        logger.debug({ initialAudioNodeConnection, label }, 'Initial AudioNodeConnection');
+        const audioGraph = createAudioGraphProxy(createAudioGraph(initialAudioNodeConnection, props.audioGraphOptions), {
+            connect: (target, args) => {
+                logger.debug({ label, target, args }, 'connect nodes');
+            },
+            disconnect: (target, args) => {
+                logger.debug({ label, target, args }, 'disconnect nodes');
+            },
+        });
+        props.audioGraph = audioGraph;
+        const denoiseNode = await createDenoiseNode(props.denoise);
+        const connectDenoise = (node) => {
+            if (node) {
+                audioGraph.disconnect([source, ...otherNodes, destination]);
+                audioGraph.connect([source, node, ...otherNodes, destination]);
+            }
+        };
+        connectDenoise(denoiseNode);
+        const track = destination?.node?.stream.getAudioTracks().at(0) ?? null;
+        assert(track, 'No audio track available');
+        // Inherit contentHint from previous track
+        track.contentHint = prevMediaTrack.track.contentHint;
+        const release = async () => {
+            logger.debug({ label }, 'Release Media');
+            track.stop();
+            // Release Props
+            await audioGraph.release();
+            props.denoiseNode = undefined;
+            props.analyzer = undefined;
+            props.audioGraph = undefined;
+        };
+        const mute = (mute) => {
+            if (track) {
+                track.enabled = !mute;
+            }
+        };
+        return createMediaTrack({
+            label,
+            kind: 'audioinput',
+            constraints: prevMediaTrack.getConstraints(),
+            previousMediaTrack: prevMediaTrack,
+            input: prevMediaTrack.input,
+            expectedInput: prevMediaTrack.expectedInput,
+            mute,
+            track,
+            release,
+            applyConstraints: async (constraints) => {
+                if (isEmpty(constraints)) {
                     return;
                 }
-                const features = updateFeatureProps(constraints.audio, props);
-                logger.debug({ scope, constraints: constraints.audio, features }, 'apply audio constraints');
+                const features = updateFeatureProps(constraints, props);
+                logger.debug({ label, constraints: constraints, features }, 'apply audio constraints');
                 if (isEmpty(features) ||
                     ['closed', 'closing'].includes(audioGraph.state)) {
                     return;
                 }
-                try {
-                    const denoiseNode = await createDenoiseNode(props.denoise);
-                    if (denoiseNode) {
-                        if (!source.hasConnectedTo(denoiseNode)) {
-                            connectDenoise(denoiseNode);
-                        }
-                    }
-                    else {
-                        if (props.denoiseNode) {
-                            audioGraph.disconnect([
-                                source,
-                                props.denoiseNode,
-                                ...otherNodes,
-                                destination,
-                            ]);
-                            audioGraph.connect([
-                                source,
-                                ...otherNodes,
-                                destination,
-                            ]);
-                            audioGraph.releaseInit(props.denoiseNode);
-                            props.denoiseNode = undefined;
-                        }
-                    }
-                    const analyzer = createAnalyzer();
-                    if (analyzer) {
-                        audioGraph.connect([source, analyzer]);
+                const denoiseNode = await createDenoiseNode(props.denoise);
+                if (denoiseNode) {
+                    if (!source.hasConnectedTo(denoiseNode)) {
+                        connectDenoise(denoiseNode);
                     }
-                    else {
-                        if (props.analyzer) {
-                            audioGraph.disconnect([source, props.analyzer]);
-                            audioGraph.releaseInit(props.analyzer);
-                            props.analyzer = undefined;
-                        }
+                }
+                else {
+                    if (props.denoiseNode) {
+                        audioGraph.disconnect([
+                            source,
+                            props.denoiseNode,
+                            ...otherNodes,
+                            destination,
+                        ]);
+                        audioGraph.connect([
+                            source,
+                            ...otherNodes,
+                            destination,
+                        ]);
+                        audioGraph.releaseInit(props.denoiseNode);
+                        props.denoiseNode = undefined;
                     }
                 }
-                catch (error) {
-                    if (error instanceof Error) {
-                        logger.error({
-                            scope,
-                            constraints: constraints.audio,
-                            features,
-                        }, 'failed to apply audio constraints');
+                const analyzer = createAnalyzer();
+                if (analyzer) {
+                    audioGraph.connect([source, analyzer]);
+                }
+                else {
+                    if (props.analyzer) {
+                        audioGraph.disconnect([source, props.analyzer]);
+                        audioGraph.releaseInit(props.analyzer);
+                        props.analyzer = undefined;
                     }
                 }
-            });
-            const release = async () => {
-                logger.debug({ scope }, 'Release Media');
-                stopMediaStream(stream);
-                // Release Props
-                await audioGraph.release();
-                await media.release();
-                props.denoiseNode = undefined;
-                props.analyzer = undefined;
-                props.audioGraph = undefined;
-            };
-            const muteAudio = (mute) => {
-                media.muteAudio(mute);
-                muteStreamTrack(stream)(mute, 'audio');
-            };
-            const muteVideo = (mute) => {
-                media.muteVideo(mute);
-                muteStreamTrack(stream)(mute, 'video');
-            };
-            const prevGetSettings = media.getSettings;
-            return wrapToJSON(shallowCopy(media, {
-                stream,
-                applyConstraints,
-                muteAudio,
-                muteVideo,
-                release,
-                getSettings: () => {
-                    const { audio, video } = prevGetSettings();
-                    const denoise = !!props.denoiseNode &&
-                        source.hasConnectedTo(props.denoiseNode);
-                    const asd = !!props.asd;
-                    const vad = !!props.vad;
-                    const contentHint = stream.getAudioTracks().at(0)
-                        ?.contentHint ?? '';
-                    const audioSettings = {
-                        denoise,
-                        asd,
-                        vad,
-                        contentHint,
-                    };
-                    const settings = {
-                        audio: audio.map(settings => ({
-                            ...settings,
-                            ...audioSettings,
-                        })),
-                        video,
-                    };
-                    logger.debug({ scope, settings: audioSettings }, 'get audio processor settings');
-                    return settings;
-                },
-            }));
-        }
-        catch (error) {
-            logger.error({ scope, error }, 'Unable to use WebAudio, return the raw media instead');
-            return media;
-        }
+            },
+            getSettings: () => {
+                const denoise = !!props.denoiseNode &&
+                    source.hasConnectedTo(props.denoiseNode);
+                const asd = !!props.asd;
+                const vad = !!props.vad;
+                const contentHint = track.contentHint;
+                const audioSettings = {
+                    denoise,
+                    asd,
+                    vad,
+                    contentHint,
+                };
+                return audioSettings;
+            },
+        });
     };
 };

package/dist/constants.d.ts

@@ -0,0 +1,7 @@
+export declare const PROCESSOR_LABELS: Readonly<{
+    GetUserMedia: "GetUserMedia";
+    VideoProcessor: "VideoProcessor";
+    AudioProcessor: "AudioProcessor";
+    AudioMixingProcessor: "AudioMixingProcessor";
+}>;
+export declare const GET_USER_MEDIA_TIMEOUT_MS = 10000;

package/dist/constants.js

@@ -0,0 +1,7 @@
+export const PROCESSOR_LABELS = Object.freeze({
+    GetUserMedia: 'GetUserMedia',
+    VideoProcessor: 'VideoProcessor',
+    AudioProcessor: 'AudioProcessor',
+    AudioMixingProcessor: 'AudioMixingProcessor',
+});
+export const GET_USER_MEDIA_TIMEOUT_MS = 10000;

package/dist/media.d.ts

@@ -1,8 +1,18 @@
-import type { MediaOptions, MediaController } from './types';
+import type { IndexedDevices, MediaDeviceRequest, InputDevicePermission } from '@pexip/media-control';
+import type { GetUserMediaProcess, Media, MediaController, MediaOptions, MediaSignals, MediaTrack } from './types';
+export interface MediaUpdaterOptions {
+    getUserMedia: GetUserMediaProcess;
+    getCurrentDevices: () => Promise<IndexedDevices>;
+    shouldDiscardMedia: () => boolean;
+    onMediaTracksChanged: (media: Media, tracks: MediaTrack[]) => void;
+    getInputDevicePermission?: () => Promise<InputDevicePermission>;
+    signals?: MediaSignals;
+}
+export declare const createMediaUpdater: ({ getUserMedia, getCurrentDevices, shouldDiscardMedia: shouldDisgardMedia, onMediaTracksChanged, getInputDevicePermission, signals, }: MediaUpdaterOptions) => (constraints: MediaDeviceRequest, currentMedia: Media | undefined) => Promise<void>;
 /**
  * Create an object to interact with the media scream, which is usually used for
  * our main stream.
  *
  * @param options - @see MediaOptions
  */
-export declare const createMedia: ({ getMuteState, signals, mediaProcessors, getDefaultConstraints, }: MediaOptions) => MediaController;
+export declare const createMedia: ({ getMuteState, signals, audioProcessors, videoProcessors, getDefaultConstraints, }: MediaOptions) => MediaController;