@pexip/media 18.4.0 → 19.0.0

package/dist/userMedia.js

@@ -1,9 +1,10 @@
-import { MediaDeviceFailure, applyConstraints, findMediaInputFromMediaStreamTrack, getDevices, getUserMedia, isExactDeviceConstraint, isRequestedResolution, isStreamingRequestedDevices, muteStreamTrack, stopMediaStream, isAudioInput, isVideoInput, hasAnyInputs, hasAnyGrantedInput, } from '@pexip/media-control';
-import { isEmpty } from '@pexip/utils';
+import { MediaDeviceFailure, extractConstraintsWithKeys, findMediaInputFromMediaStreamTrack, getUserMedia, isExactDeviceConstraint, isStreamingRequestedDevices, } from '@pexip/media-control';
+import { assert } from '@pexip/utils';
 import { UserMediaStatus } from './types';
-import { shallowCopy, makeDeriveDeviceStatus, createMemorizedGetExpectedInput, isMuted, wrapToJSON, } from './utils';
+import { makeDeriveDeviceStatus, buildMedia, createMediaTrack, findExpectedInput, } from './utils';
 import { logger } from './logger';
 import { isUnknownError } from './status';
+import { PROCESSOR_LABELS } from './constants';
 export const toErrorDeviceStatus = (error, deriveDeviceStatus) => {
     switch (error.message) {
         case MediaDeviceFailure.AudioInputDeviceNotFoundError:
@@ -13,7 +14,6 @@ export const toErrorDeviceStatus = (error, deriveDeviceStatus) => {
         case MediaDeviceFailure.AudioAndVideoDeviceNotFoundError:
             return UserMediaStatus.AudioVideoDevicesNotFound;
         case MediaDeviceFailure.NotAllowedError:
-        case MediaDeviceFailure.NotFoundError:
         case MediaDeviceFailure.SecurityError:
         case MediaDeviceFailure.PermissionDeniedError:
             return deriveDeviceStatus(UserMediaStatus.PermissionsRejectedAudioInput, UserMediaStatus.PermissionsRejectedVideoInput, UserMediaStatus.PermissionsRejected);
@@ -22,9 +22,9 @@ export const toErrorDeviceStatus = (error, deriveDeviceStatus) => {
         case MediaDeviceFailure.AbortError:
             return deriveDeviceStatus(UserMediaStatus.AudioDeviceInUse, UserMediaStatus.VideoDeviceInUse, UserMediaStatus.DevicesInUse);
         case MediaDeviceFailure.MissingConstraintsError:
-            return deriveDeviceStatus(UserMediaStatus.NoAudioDevicesFound, UserMediaStatus.NoVideoDevicesFound, UserMediaStatus.NoDevicesFound);
         case MediaDeviceFailure.OverconstrainedError:
-            return deriveDeviceStatus(UserMediaStatus.AudioOverconstrained, UserMediaStatus.VideoOverconstrained, UserMediaStatus.Overconstrained);
+        case MediaDeviceFailure.NotFoundError:
+            return deriveDeviceStatus(UserMediaStatus.NoAudioDevicesFound, UserMediaStatus.NoVideoDevicesFound, UserMediaStatus.NoDevicesFound);
         case MediaDeviceFailure.TypeError:
             return deriveDeviceStatus(UserMediaStatus.InvalidAudioConstraints, UserMediaStatus.InvalidVideoConstraints, UserMediaStatus.InvalidConstraints);
         case MediaDeviceFailure.NotSupportedError:
@@ -34,7 +34,10 @@ export const toErrorDeviceStatus = (error, deriveDeviceStatus) => {
     }
 };
 export const mapErrorStatus = (status) => (constraints) => {
-    const toStatus = makeDeriveDeviceStatus(constraints);
+    const toStatus = makeDeriveDeviceStatus({
+        audio: constraints.audio ?? false,
+        video: constraints.video ?? false,
+    });
     switch (status) {
         case UserMediaStatus.AudioVideoDevicesNotFound:
             return toStatus(UserMediaStatus.AudioDeviceNotFound, UserMediaStatus.VideoDeviceNotFound, status);
@@ -69,12 +72,7 @@ export const toSameDeviceStatus = ({ audio, video, }) => {
     return UserMediaStatus.PermissionsGrantedFallback;
 };
 export const toOnlyDeviceStatus = (kind, matched, devices) => {
-    const hasRelatedDevices = devices.some(d => {
-        if (kind === 'audioinput') {
-            return d.kind === 'videoinput';
-        }
-        return d.kind === 'audioinput';
-    });
+    const hasRelatedDevices = devices.size(kind) > 0;
     if (matched) {
         if (hasRelatedDevices) {
             return kind === 'audioinput'
@@ -117,14 +115,11 @@ export const deriveErrorLevel = (error, status, constraints) => {
         ? 'error'
         : 'warn';
 };
-export const requestUserMedia = (currentDevices, getMedia = getUserMedia) => async (constraints) => {
+export const requestUserMedia = (getCurrentDevices, getMedia = getUserMedia) => async (constraints) => {
     const deriveDeviceStatus = makeDeriveDeviceStatus(constraints);
-    const devices = currentDevices ?? (await getDevices());
     try {
         const stream = await getMedia(constraints);
-        const grantedDevices = hasAnyGrantedInput(devices)
-            ? devices
-            : await getDevices();
+        const grantedDevices = await getCurrentDevices();
         const { audio, video } = isStreamingRequestedDevices(constraints, stream, grantedDevices);
         const onlyAudioStatus = toOnlyDeviceStatus('audioinput', audio, grantedDevices);
         const onlyVideoStatus = toOnlyDeviceStatus('videoinput', video, grantedDevices);
@@ -155,179 +150,128 @@ export const mergeNoDeviceStatus = (constraints, anyDevices, status) => {
     }
     return status;
 };
-export const requestUserMediaWithRetry = (currentDevices, createRequestUserMedia = requestUserMedia) => async (constraints) => {
-    const devices = isEmpty(currentDevices)
-        ? await getDevices()
-        : currentDevices;
-    const [anyAudioDevices, anyVideoDevices] = hasAnyInputs(devices);
-    const request = (constraints, devices) => createRequestUserMedia(devices)(constraints);
-    if (anyAudioDevices && anyVideoDevices) {
-        const [stream, status] = await request(constraints, devices);
-        if (stream) {
-            return [stream, status];
-        }
-        const mapStatus = mapErrorStatus(status);
-        if (constraints.video) {
-            const [audioStream] = await request({
-                ...constraints,
-                video: false,
-            }, devices);
-            if (audioStream) {
-                // When success, we know it is video related error
-                const videoStatus = mapStatus({ video: true });
-                return [audioStream, videoStatus];
-            }
-        }
-        if (constraints.audio) {
-            const [videoStream] = await request({
-                ...constraints,
-                audio: false,
-            }, devices);
-            if (videoStream) {
-                // When success, we know it is audio related error
-                return [videoStream, mapStatus({ audio: true })];
-            }
-        }
-        return [stream, status];
-    }
-    else {
-        const [stream, status] = await request({
-            audio: anyAudioDevices && constraints.audio,
-            video: anyVideoDevices && constraints.video,
-        }, devices);
+export const requestUserMediaWithRetry = (getCurrentDevices, createRequestUserMedia = requestUserMedia, gUM = getUserMedia) => {
+    const request = createRequestUserMedia(getCurrentDevices, gUM);
+    return async (constraints) => {
+        const [stream, status] = await request(constraints);
+        const devices = await getCurrentDevices();
+        const anyAudioDevices = devices.size('audioinput') > 0;
+        const anyVideoDevices = devices.size('videoinput') > 0;
         return [
             stream,
             mergeNoDeviceStatus(constraints, { audio: anyAudioDevices, video: anyVideoDevices }, status),
         ];
-    }
+    };
 };
 /**
  * A process to get user media
  */
-export const createGetUserMediaProcess = (getUserMedia, getCurrentDevices, { initialMedia, scope = 'media' }) => {
-    const props = {
-        unsubscribe: undefined,
-        currentDevices: getCurrentDevices(),
-        cachedAudioTrackId: '',
-        cachedVideoTrackId: '',
-        cachedAudioInput: undefined,
-        cachedVideoInput: undefined,
-    };
-    const getExpectedAudioInput = createMemorizedGetExpectedInput();
-    const getExpectedVideoInput = createMemorizedGetExpectedInput();
-    /**
-     * Memorized get media input
-     */
-    const getInput = ([track] = []) => {
-        if (!track) {
-            return undefined;
+export const createGetUserMediaProcess = ({ getUserMedia, getCurrentDevices, signals, scope = 'media', }) => {
+    return async ({ constraints, permission, originalConstraints, currentMedia, }) => {
+        // Release the current track(s)
+        if (currentMedia) {
+            await Promise.all(currentMedia.getTracks().map(track => {
+                switch (track.kind) {
+                    case 'audioinput':
+                        // When the constraints is undefined we should do nothing
+                        if (constraints.audio !== undefined) {
+                            // There is only one audio track, so we can safely assign it
+                            currentMedia.removeTrack(track);
+                            return track.release();
+                        }
+                        return Promise.resolve();
+                    case 'videoinput':
+                        // When the constraints is undefined we should do nothing
+                        if (constraints.video !== undefined) {
+                            // There is only one video track, so we can safely assign it
+                            currentMedia.removeTrack(track);
+                            return track.release();
+                        }
+                        return Promise.resolve();
+                    default:
+                        assert(false, `Undexpected track kind: ${track.kind}`);
+                }
+            }));
         }
-        const isAudio = track.kind === 'audio';
-        const cacheKey = isAudio ? 'cachedAudioInput' : 'cachedVideoInput';
-        const cachedTrackId = isAudio
-            ? 'cachedAudioTrackId'
-            : 'cachedVideoTrackId';
-        const trackId = track.label;
-        const { width, height } = track.getSettings();
-        if (trackId === props[cachedTrackId] &&
-            track.readyState === 'live' &&
-            (isAudio ||
-                (!isAudio &&
-                    isRequestedResolution({ width, height }, props[cacheKey])))) {
-            return props[cacheKey];
+        const currentDevices = await getCurrentDevices();
+        logger.debug({
+            scope,
+            originalConstraints,
+            constraints,
+            currentDevices,
+            currentMedia,
+        }, 'pre getUserMedia');
+        if (!constraints.audio && !constraints.video) {
+            currentMedia?.setOriginalConstraints(originalConstraints);
+            return (currentMedia ??
+                buildMedia({
+                    constraints,
+                    permission,
+                    originalConstraints,
+                    devices: currentDevices,
+                    status: UserMediaStatus.PermissionsGranted,
+                    stream: undefined,
+                    signals,
+                    tracks: [],
+                }));
         }
-        const devices = getCurrentDevices();
-        const findTrack = findMediaInputFromMediaStreamTrack(devices);
-        const input = findTrack(track);
-        // Update cache
-        props[cacheKey] = input;
-        props[cachedTrackId] = trackId;
-        return input;
-    };
-    return async (constraints) => {
         const [stream, status] = await getUserMedia(constraints);
-        logger.debug({ scope, stream, status }, 'getUserMedia');
-        props.currentDevices = await getDevices();
-        const deriveDeviceStatus = makeDeriveDeviceStatus(constraints);
-        const muteTrack = muteStreamTrack(stream);
-        const muteAudio = (mute) => muteTrack(mute, 'audio');
-        const muteVideo = (mute) => muteTrack(mute, 'video');
-        return wrapToJSON(shallowCopy(initialMedia ?? {}, {
+        logger.debug({ scope, stream, status }, 'post getUserMedia');
+        const devices = await getCurrentDevices();
+        const findInput = findMediaInputFromMediaStreamTrack(devices);
+        let audioMediaTrack;
+        let videoMediaTrack;
+        const extractContentHint = extractConstraintsWithKeys(['contentHint']);
+        if (constraints.audio !== undefined) {
+            const audioTrack = stream?.getAudioTracks().at(0);
+            const audioInput = findInput(audioTrack);
+            const { contentHint: [[contentHint] = []], } = extractContentHint(constraints.audio);
+            if (audioTrack) {
+                audioTrack.contentHint = contentHint ?? '';
+            }
+            audioMediaTrack = createMediaTrack({
+                label: PROCESSOR_LABELS.GetUserMedia,
+                kind: 'audioinput',
+                track: audioTrack,
+                input: audioInput,
+                expectedInput: findExpectedInput(devices, constraints.audio, audioInput, 'audioinput'),
+                constraints: constraints.audio,
+                signals,
+            });
+            currentMedia?.addTrack(audioMediaTrack);
+        }
+        if (constraints.video !== undefined) {
+            const videoTrack = stream?.getVideoTracks().at(0);
+            const videoInput = findInput(videoTrack);
+            const { contentHint: [[contentHint] = []], } = extractContentHint(constraints.video);
+            if (videoTrack) {
+                videoTrack.contentHint = contentHint ?? '';
+            }
+            videoMediaTrack = createMediaTrack({
+                label: PROCESSOR_LABELS.GetUserMedia,
+                kind: 'videoinput',
+                track: videoTrack,
+                input: videoInput,
+                expectedInput: findExpectedInput(devices, constraints.video, videoInput, 'videoinput'),
+                constraints: constraints.video,
+                signals,
+            });
+            currentMedia?.addTrack(videoMediaTrack);
+        }
+        if (currentMedia) {
+            currentMedia.status = status;
+            currentMedia.setOriginalConstraints(originalConstraints);
+            return currentMedia;
+        }
+        return buildMedia({
             constraints,
-            rawStream: stream,
+            permission,
+            originalConstraints,
+            devices,
             stream,
             status,
-            get devices() {
-                return props.currentDevices;
-            },
-            set devices(newDevices) {
-                props.currentDevices = newDevices;
-            },
-            get audioInput() {
-                return getInput(stream?.getAudioTracks());
-            },
-            get videoInput() {
-                return getInput(stream?.getVideoTracks());
-            },
-            get expectedAudioInput() {
-                return getExpectedAudioInput(constraints.audio, () => ({
-                    devices: getCurrentDevices().filter(isAudioInput),
-                    input: getInput(stream?.getAudioTracks()),
-                }));
-            },
-            get expectedVideoInput() {
-                return getExpectedVideoInput(constraints.video, () => ({
-                    devices: getCurrentDevices().filter(isVideoInput),
-                    input: getInput(stream?.getVideoTracks()),
-                }));
-            },
-            muteAudio,
-            muteVideo,
-            get audioMuted() {
-                return isMuted(stream?.getAudioTracks());
-            },
-            get videoMuted() {
-                return isMuted(stream?.getVideoTracks());
-            },
-            applyConstraints: async (constraints) => {
-                try {
-                    logger.debug({ scope, constraints: constraints }, 'apply native constraints');
-                    return await applyConstraints(stream?.getTracks(), constraints);
-                }
-                catch (error) {
-                    if (error instanceof Error) {
-                        const status = toErrorDeviceStatus(error, deriveDeviceStatus);
-                        logger.error({
-                            scope,
-                            status,
-                            error,
-                            constraints: constraints,
-                        }, 'fail to apply native constraints');
-                        initialMedia.status = status;
-                    }
-                    throw error;
-                }
-            },
-            getSettings: () => {
-                const settings = {
-                    audio: stream
-                        ?.getAudioTracks()
-                        .map(track => track.getSettings()) ?? [],
-                    video: stream
-                        ?.getVideoTracks()
-                        .map(track => track.getSettings()) ?? [],
-                };
-                logger.debug({ settings, scope }, 'get native settings');
-                return settings;
-            },
-            release: () => {
-                stopMediaStream(stream);
-                if (props.unsubscribe) {
-                    props.unsubscribe();
-                    props.unsubscribe = undefined;
-                }
-                return Promise.resolve();
-            },
-        }));
+            signals,
+            tracks: [audioMediaTrack, videoMediaTrack].flatMap(track => track ? [track] : []),
+        });
     };
 };

package/dist/utils.d.ts

@@ -1,26 +1,14 @@
-import type { MediaDeviceInfoLike, MediaDeviceRequest } from '@pexip/media-control';
-import { UserMediaStatus } from './types';
-import type { Media, MediaAttributes, Pipeline, Process, ProcessMedia, ExtendedMediaTrackSettingsKey, ExtendedMediaTrackSettings } from './types';
+import type { IndexedDevices, InputDeviceConstraint, MediaDeviceInfoLike, MediaDeviceRequest } from '@pexip/media-control';
+import type { ExtendedMediaTrackSettings, ExtendedMediaTrackSettingsKey, Media, MediaInit, MediaTrack, MediaTrackInit, TrackProcessor, UserMediaStatus } from './types';
 export declare const makeDeriveDeviceStatus: (constraints: MediaDeviceRequest) => (audio: UserMediaStatus, video: UserMediaStatus, both: UserMediaStatus) => UserMediaStatus;
-export declare const createMediaProcess: (process: ProcessMedia) => Process<Promise<Media>>;
-export declare const createMediaPipeline: <T = MediaDeviceRequest>(init: Pipeline<T> | (() => Pipeline<T>)) => {
-    pipe: (process: Process<Promise<Media>>) => void;
-    execute: (m: T) => Promise<Media>;
-};
+export declare const createMediaTrack: (trackInit: MediaTrackInit) => MediaTrack;
+export declare const createTrackProcessingPipeline: (processors: TrackProcessor[]) => (track: MediaTrack) => Promise<MediaTrack>;
 /**
  * Interpret provided input to resolve to a MediaDeviceInfoLike when possible
  * otherwise `undefined`
  */
 export declare const interpretInput: (input: boolean | MediaDeviceInfoLike | undefined, getCurrentInput: () => MediaDeviceInfoLike | undefined) => MediaDeviceInfoLike | undefined;
-type InputConstraints = MediaDeviceRequest['audio'];
-interface MediaInputInfo {
-    devices: MediaDeviceInfoLike[];
-    input: MediaDeviceInfoLike | undefined;
-}
-/**
- * Memorized Expected Input
- */
-export declare const createMemorizedGetExpectedInput: () => (constraints: InputConstraints, getInfo: () => MediaInputInfo) => ExpectedInput;
+export declare const findExpectedInput: (devices: IndexedDevices, constraints: InputDeviceConstraint | undefined, input: MediaDeviceInfoLike | undefined, kind: 'audioinput' | 'videoinput') => MediaDeviceInfoLike | undefined;
 /**
  * A utility function to check if the provided track is muted. There are
  * 2 factors to be considered: `MediaStreamTrack['muted']` and `MediaStreamTrack['enabled']`.
@@ -32,19 +20,14 @@ export declare const createMemorizedGetExpectedInput: () => (constraints: InputC
  * |     false       | false | true  |
  * ```
  *
- * @param tracks - The tracks can be got from `MediaStream['getAudioStats']` or
+ * @param track - The tracks can be got from `MediaStream['getAudioStats']` or
  * `MediaStream['getVideoTracks']`
  *
  * @returns `true` means muted, `false` means not muted and `undefined` means
  * there is no track to check
  */
-export declare const isMuted: (tracks: MediaStreamTrack[] | undefined) => boolean | undefined;
-type ExpectedInput = MediaDeviceInfoLike | undefined;
-export declare const buildMedia: (getMedia: () => Partial<Media>, onSetStatus?: ((status: UserMediaStatus) => void) | undefined) => Media;
-/**
- * Clone the media from the rawStream (if any), otherwise, stream
- */
-export declare const cloneMedia: (media: Media) => Promise<Media>;
+export declare const isTrackMuted: (track: MediaStreamTrack | undefined) => boolean | undefined;
+export declare const buildMedia: (mediaInit: MediaInit, onDevicesChanged?: import("@pexip/signal").Signal<IndexedDevices>) => Media;
 /**
  * Shallow copy the provided object and override with provided overriding
  *
@@ -61,32 +44,12 @@ export declare const getDevicesChanges: (prev: MediaDeviceInfoLike[], next: Medi
     lost: MediaDeviceInfoLike[];
     devices: MediaDeviceInfoLike[];
 };
-/**
- * Apply Extended constraints on top of the original
- *
- * @param media - The media from the media pipeline
- * @param applyExtended - The function to be called when the previous
- * `applyConstraints` is done
- */
-export declare const applyExtendedConstraints: (media: Media, applyExtended: (constraints: MediaDeviceRequest) => Promise<void>) => (constraints: MediaDeviceRequest) => Promise<void>;
 export declare const AUDIO_SETTINGS_KEYS: ExtendedMediaTrackSettingsKey[];
 export declare const VIDEO_SETTINGS_KEYS: ExtendedMediaTrackSettingsKey[];
 export declare const MIXING_SETTINGS_KEYS: ExtendedMediaTrackSettingsKey[];
 export declare const hasSettingsChanged: (keysToLookFor: ExtendedMediaTrackSettingsKey[]) => (settingsA: ExtendedMediaTrackSettings | undefined, settingsB: ExtendedMediaTrackSettings | undefined) => boolean;
-export declare const toJSON: (media: Partial<MediaAttributes>) => {
-    constraints: MediaDeviceRequest | undefined;
-    devices: MediaDeviceInfoLike[] | undefined;
-    stream: MediaStream | undefined;
-    rawStream: MediaStream | undefined;
-    audioInput: MediaDeviceInfoLike | undefined;
-    videoInput: MediaDeviceInfoLike | undefined;
-    expectedAudioInput: MediaDeviceInfoLike | undefined;
-    expectedVideoInput: MediaDeviceInfoLike | undefined;
-    status: UserMediaStatus | undefined;
-    audioMuted: boolean | undefined;
-    videoMuted: boolean | undefined;
-};
-export declare const wrapToJSON: (media: Media) => Media;
+export declare const diffSettings: (keysToLookFor: ExtendedMediaTrackSettingsKey[]) => (settingsA: ExtendedMediaTrackSettings | undefined, settingsB: ExtendedMediaTrackSettings | undefined) => ExtendedMediaTrackSettings | undefined;
+export declare const mergeSettings: (settingsA: ExtendedMediaTrackSettings | undefined, settingsB: ExtendedMediaTrackSettings | undefined) => ExtendedMediaTrackSettings | undefined;
 /**
  * A function to get the blur kernel size of image height
  *
@@ -110,4 +73,19 @@ export declare const applyContentHint: <T extends "" | "speech" | "speech-recogn
  * @returns true if the browser supports PTZ feature, otherwise false
  */
 export declare const hasPtzFeature: () => boolean;
+interface RequestState {
+    kind: 'audioinput' | 'videoinput';
+    permission: PermissionState;
+    currentDevices: IndexedDevices;
+    request: InputDeviceConstraint | undefined;
+    currentMediaTracks: MediaTrack[];
+    force?: boolean;
+}
+export declare const refineMediaConstraints: (state: RequestState) => InputDeviceConstraint | undefined;
+interface ProcessorParams {
+    audioProcessors: TrackProcessor[];
+    videoProcessors: TrackProcessor[];
+    onProcessingError(error: Error, track: MediaTrack): void;
+}
+export declare const createMediaProcessor: ({ audioProcessors, videoProcessors, onProcessingError, }: ProcessorParams) => (newTracks: MediaTrack[]) => Promise<MediaTrack[]>;
 export {};