@pexip/media 17.2.0 → 17.4.0

package/dist/index.js

@@ -0,0 +1,11 @@
+export { createMedia } from './media';
+export { createAudioStreamProcess } from './audioProcessor';
+export { createVideoStreamProcess } from './videoProcessor';
+export * from './types';
+export { setLogger } from './logger';
+export * from './previewController';
+export * from './status';
+export * from './signals';
+export { applyContentHint } from './utils';
+export { createAudioMixingProcess } from './audioMixingProcessor';
+export * from './displayMedia';

package/dist/logger.d.ts

@@ -0,0 +1,11 @@
+import type { Logger } from './baseLogger';
+export declare let logger: Readonly<Logger>;
+export declare function setLogger(newLogger: Logger): void;
+/**
+ * A logger wrapper to always include some meta base data to the log
+ *
+ * @param metaBase - The meta data which will always be included into the log
+ */
+export declare const createModuleLogger: (metaBase: unknown) => Logger;
+export declare const createLogProxyHandler: <T extends object>(logger: Logger, name: string, scope: string) => ProxyHandler<T>;
+export declare const proxyWithLog: (logger: Logger, scope: string) => <T extends object>(obj: T, name: string) => T;

package/dist/logger.js

@@ -0,0 +1,44 @@
+import { createConsoleLogger } from './baseLogger';
+export let logger = createConsoleLogger();
+export function setLogger(newLogger) {
+    logger = newLogger;
+}
+/**
+ * A logger wrapper to always include some meta base data to the log
+ *
+ * @param metaBase - The meta data which will always be included into the log
+ */
+export const createModuleLogger = (metaBase) => {
+    return Object.keys(logger).reduce((log, key) => {
+        const k = key;
+        log[k] = (meta, msg) => {
+            if (typeof meta === 'string') {
+                return logger[k](metaBase, meta);
+            }
+            if (typeof meta === 'object') {
+                return logger[k]({ ...meta, meta: metaBase }, msg);
+            }
+            return logger[k]({ meta: metaBase, context: meta }, msg);
+        };
+        return log;
+    }, {});
+};
+export const createLogProxyHandler = (logger, name, scope) => {
+    return {
+        get: (target, p, receiver) => {
+            const value = Reflect.get(target, p, receiver);
+            logger.debug({ scope, name, prop: p, value }, `called get ${name}[${String(p)}]`);
+            return value;
+        },
+        // eslint-disable-next-line max-params --- from lib.es2015 and log
+        set: (target, p, value, receiver) => {
+            logger.debug(
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- it's a log call...
+            { scope, name, prop: p, value }, `called set ${name}[${String(p)}]`);
+            return Reflect.set(target, p, value, receiver);
+        },
+    };
+};
+export const proxyWithLog = (logger, scope) => (obj, name) => {
+    return new Proxy(obj, createLogProxyHandler(logger, name, scope));
+};

package/dist/media.d.ts

@@ -0,0 +1,8 @@
+import type { MediaOptions, MediaController } from './types';
+/**
+ * Create an object to interact with the media scream, which is usually used for
+ * our main stream.
+ *
+ * @param options - @see MediaOptions
+ */
+export declare const createMedia: ({ getMuteState, signals, mediaProcessors, getDefaultConstraints, }: MediaOptions) => MediaController;

package/dist/media.js

@@ -0,0 +1,296 @@
+import { subscribe, MediaEventType, MediaDeviceFailure, createStreamTrackEventSubscriptions, getDevices, isRequestedResolution, mergeConstraints, shouldRequestDevice, } from '@pexip/media-control';
+import { createAsyncQueue, isEmpty } from '@pexip/utils';
+import { getPermissionStatus, deriveInitialPermissionStatus, isInitialPermissions, isInitialPermissionsGranted, } from './status';
+import { UserMediaStatus } from './types';
+import { createModuleLogger, logger } from './logger';
+import { createGetUserMediaProcess, requestUserMediaWithRetry, } from './userMedia';
+import { createMediaPipeline, createMediaProcess, buildMedia, getDevicesChanges, shallowCopy, wrapToJSON, hasSettingsChanged, AUDIO_SETTINGS_KEYS, VIDEO_SETTINGS_KEYS, MIXING_SETTINGS_KEYS, applyContentHint, } from './utils';
+import { isMedia } from './typeGuard';
+import { updateFeatureProps as getVideoFeatures } from './videoProcessor';
+import { updateFeatureProps as getAudioFeatures } from './audioProcessor';
+import { updateFeatureProps as getMixingFeatures } from './audioMixingProcessor';
+/**
+ * Proxy handler for Media Props
+ */
+const createMediaPropsHandler = (signals) => ({
+    get: (target, p) => {
+        switch (p) {
+            default:
+                return target[p];
+        }
+    },
+    set: (target, p, value) => {
+        if (target[p] === value) {
+            return true;
+        }
+        if (p === 'devices') {
+            const nextDevices = value;
+            const changes = getDevicesChanges(target[p].flatMap(device => (device.label ? [device] : [])), nextDevices);
+            if (isEmpty(changes.found) && isEmpty(changes.lost)) {
+                return true;
+            }
+        }
+        logger.debug({
+            oldValue: target[p],
+            newValue: value,
+            meta: {
+                module: 'Media',
+                props: target,
+            },
+        }, `Update Props[${p}]`);
+        switch (p) {
+            case 'devices': {
+                if (!Array.isArray(value)) {
+                    return false;
+                }
+                const devices = value;
+                target[p] = devices;
+                signals?.onDevicesChanged?.emit(devices);
+                target.media.devices = devices;
+                return true;
+            }
+            case 'media': {
+                if (!isMedia(value)) {
+                    return false;
+                }
+                const currentStatus = target[p].status;
+                target[p] = value;
+                signals?.onMediaChanged?.emit(value);
+                if (currentStatus !== value.status) {
+                    signals?.onStatusChanged?.emit(value.status);
+                }
+                return true;
+            }
+            default: {
+                Reflect.set(target, p, value);
+                return true;
+            }
+        }
+    },
+});
+/**
+ * Create an object to interact with the media scream, which is usually used for
+ * our main stream.
+ *
+ * @param options - @see MediaOptions
+ */
+export const createMedia = ({ getMuteState, signals, mediaProcessors, getDefaultConstraints = () => ({}), }) => {
+    const initMedia = (status = UserMediaStatus.Initial, devices = [], constraints) => buildMedia(() => ({ status, devices, constraints }), signals.onStatusChanged?.emit);
+    const _props = {
+        devices: [],
+        media: initMedia(),
+        discardMedia: false,
+    };
+    const props = new Proxy(_props, createMediaPropsHandler(signals));
+    const queue = createAsyncQueue();
+    const logger = createModuleLogger({
+        module: 'Media',
+        props: _props,
+        get mediaTracks() {
+            return _props.media?.stream?.getTracks();
+        },
+        get rawMediaTracks() {
+            return _props.media?.rawStream?.getTracks();
+        },
+    });
+    const cleanup = async () => {
+        if (isInitialPermissions(props.media.status)) {
+            props.discardMedia = true;
+        }
+        // carry the most recent status over for the next time
+        const status = await deriveInitialPermissionStatus(props.media.status);
+        props.media = initMedia(status, props.devices, props.media.constraints);
+    };
+    // Media Pipeline
+    const mediaPipeline = createMediaPipeline([
+        createGetUserMediaProcess(requestUserMediaWithRetry(), () => props.devices, { initialMedia: props.media }),
+        ...mediaProcessors,
+        createMediaProcess(media => {
+            // Subscribe the track event from raw stream
+            const trackSubscriptions = media.rawStream?.getTracks().map(track => createStreamTrackEventSubscriptions(track, {
+                ended: signals.onStreamTrackEnded?.emit,
+                mute: signals.onStreamTrackMuted?.emit,
+                unmute: signals.onStreamTrackUnmuted?.emit,
+            }));
+            const muteTrack = (tracks) => (muted) => {
+                const [track] = tracks;
+                const kind = track?.kind;
+                if (track && (kind === 'audio' || kind === 'video')) {
+                    media[kind === 'audio' ? 'muteAudio' : 'muteVideo'](muted);
+                    return tracks.forEach(track => {
+                        logger.debug({ trackInResult: track, intendToMute: muted }, `mute ${track.kind}`);
+                        signals.onStreamTrackEnabled?.emit(track);
+                    });
+                }
+                logger.warn({ tracks, kind }, 'trying to mute but no track');
+            };
+            const muteAudio = muteTrack(media.stream?.getAudioTracks() ?? []);
+            const muteVideo = muteTrack(media.stream?.getVideoTracks() ?? []);
+            // Sync mute
+            const { audio, video } = getMuteState();
+            media.audioMuted !== undefined && muteAudio(audio);
+            media.videoMuted !== undefined && muteVideo(video);
+            const release = async () => {
+                trackSubscriptions?.forEach(unsubscribe => unsubscribe());
+                await media.release();
+                await cleanup();
+            };
+            const applyConstraints = async (constraints) => {
+                await media.applyConstraints(constraints);
+                const videoFeatures = getVideoFeatures(constraints.video, {});
+                const audioFeatures = getAudioFeatures(constraints.audio, {});
+                media.stream
+                    ?.getAudioTracks()
+                    .forEach(applyContentHint(audioFeatures.contentHint));
+                media.stream
+                    ?.getVideoTracks()
+                    .forEach(applyContentHint(videoFeatures.contentHint));
+            };
+            logger.debug({ finalAudioMute: audio, finalVideoMute: video }, 'End of media pipeline');
+            return wrapToJSON(shallowCopy(media, {
+                release,
+                muteAudio,
+                muteVideo,
+                applyConstraints,
+            }));
+        }),
+    ]);
+    /**
+     * A function to update the media only when necessary, aka the current setup
+     * for the stream is different from the new constraints
+     *
+     * @param constraints - @see MediaDeviceRequest
+     */
+    const updateMedia = async (constraints) => {
+        if (!constraints.audio && !constraints.video) {
+            throw new Error(MediaDeviceFailure.MissingConstraintsError);
+        }
+        const shouldRequestAudio = shouldRequestDevice(constraints.audio, props.media?.rawStream?.getAudioTracks() ?? [], props.devices.filter(device => device.kind === 'audioinput'));
+        const shouldRequestVideo = shouldRequestDevice(constraints.video, props.media?.rawStream?.getVideoTracks() ?? [], props.devices.filter(device => device.kind === 'videoinput'));
+        const { audio: [prevAudioSettings], video: [prevVideoSettings], } = props.media.getSettings();
+        const videoFeatures = getVideoFeatures(constraints.video, {});
+        const audioFeatures = getAudioFeatures(constraints.audio, {});
+        const mixingFeatures = getMixingFeatures(constraints.audio, {});
+        const hasRequestedResolution = ['blur', 'overlay'].includes(videoFeatures.videoSegmentation ?? '') || // Skip when there is a render effect to modify the video
+            !props.devices.some(device => device.kind === 'videoinput' && device.label) || // Skip when no authorized video found
+            isRequestedResolution(constraints.video, props.media?.videoInput);
+        const audioFeaturesChanged = hasSettingsChanged(AUDIO_SETTINGS_KEYS)(prevAudioSettings, audioFeatures);
+        const videoFeaturesChanged = hasSettingsChanged(VIDEO_SETTINGS_KEYS)(prevVideoSettings, videoFeatures);
+        const mixingFeaturesChanged = hasSettingsChanged(MIXING_SETTINGS_KEYS)(prevAudioSettings, mixingFeatures);
+        const ptzFeaturesChanges = hasSettingsChanged(['pan', 'tilt', 'zoom'])(prevVideoSettings, videoFeatures);
+        logger.debug({
+            constraints,
+            stream: props.media.stream,
+            currentDevices: props.devices,
+            shouldRequestAudio,
+            shouldRequestVideo,
+            sameResolution: hasRequestedResolution,
+            audioFeaturesChanged,
+            videoFeaturesChanged,
+            mixingFeaturesChanged,
+        }, 'Is currently streaming requested stream');
+        if (shouldRequestAudio ||
+            shouldRequestVideo ||
+            ptzFeaturesChanges ||
+            !hasRequestedResolution ||
+            !props.media.stream) {
+            if (props.media) {
+                await props.media.release();
+            }
+            if (props.discardMedia) {
+                props.discardMedia = false;
+            }
+            const media = await mediaPipeline.execute(constraints);
+            media.stream
+                ?.getAudioTracks()
+                .forEach(applyContentHint(audioFeatures.contentHint));
+            media.stream
+                ?.getVideoTracks()
+                .forEach(applyContentHint(videoFeatures.contentHint));
+            if (props.discardMedia) {
+                logger.debug('Discard media');
+                return await media.release();
+            }
+            props.media = media;
+        }
+        else if (audioFeaturesChanged ||
+            videoFeaturesChanged ||
+            mixingFeaturesChanged) {
+            props.media.constraints = constraints;
+            await props.media.applyConstraints({
+                audio: audioFeatures,
+                video: videoFeatures,
+            });
+        }
+    };
+    const mergeMediaConstraints = (constraints) => {
+        const { audio, video } = getDefaultConstraints();
+        return {
+            audio: audio === false
+                ? false
+                : mergeConstraints(audio)(constraints.audio),
+            video: video === false
+                ? false
+                : mergeConstraints(video)(constraints.video),
+        };
+    };
+    const getUserMediaAsync = async (constraints) => {
+        queue.enqueue(async () => await updateMedia(mergeMediaConstraints(constraints)), false);
+        await queue.execute();
+    };
+    const getUserMedia = constraints => {
+        queue.enqueue(async () => await updateMedia(mergeMediaConstraints(constraints)));
+    };
+    const tryAndGetUserMedia = constraints => {
+        queue.enqueue(async () => {
+            props.media.status = await getPermissionStatus();
+            logger.debug({ status: props.media.status }, 'Current Permission State');
+            const mergedConstraints = mergeMediaConstraints(constraints);
+            props.media.constraints = mergedConstraints;
+            props.devices = await getDevices();
+            if (!mergedConstraints.audio && !mergedConstraints.video) {
+                return;
+            }
+            if (isInitialPermissionsGranted(props.media.status)) {
+                return await updateMedia(mergedConstraints);
+            }
+        });
+    };
+    // Media Control Event handlers
+    const handleNoInputs = (_availableDevices) => {
+        props.media.status = UserMediaStatus.NoDevicesFound;
+    };
+    const handleDeviceChange = (devices) => {
+        props.devices = devices;
+    };
+    // Subscribe Media Events for devices
+    subscribe(event => {
+        switch (event.detail.type) {
+            case MediaEventType.NoInputDevices:
+                logger.debug(event.detail, 'NoInputDevices emitted');
+                handleNoInputs(event.detail.devices);
+                break;
+            case MediaEventType.DevicesFound:
+            case MediaEventType.DevicesLost:
+                logger.debug(event.detail, 'DevicesFound/Lost emitted');
+                handleDeviceChange([
+                    ...event.detail.authorizedDevices,
+                    ...event.detail.unauthorizedDevices,
+                ]);
+                break;
+            default:
+                break;
+        }
+    });
+    return {
+        get media() {
+            return props.media;
+        },
+        get devices() {
+            return props.devices;
+        },
+        getUserMedia,
+        getUserMediaAsync,
+        tryAndGetUserMedia,
+    };
+};

package/dist/previewController.d.ts

@@ -0,0 +1,64 @@
+import type { MediaDeviceRequest, MediaDeviceInfoLike } from '@pexip/media-control';
+import type { AsyncQueueOptions } from '@pexip/utils';
+import type { Media, Unsubscribe, MediaSignals, MediaProcessor } from './types';
+type EventCallback<T> = (event: T) => void;
+type EventErrorCallback = (error: Error) => void;
+export type PreviewInput = MediaDeviceInfoLike | undefined;
+export interface PreviewEventHandler {
+    audioInput?: EventCallback<PreviewInput>;
+    videoInput?: EventCallback<PreviewInput>;
+    media?: EventCallback<Media>;
+    videoInputError?: EventErrorCallback;
+    audioInputError?: EventErrorCallback;
+    applyChangesError?: EventErrorCallback;
+    revertChangesError?: EventErrorCallback;
+    updatingPreview?: EventCallback<boolean>;
+    updatingMain?: EventCallback<boolean>;
+    unsubscribeMain?: Unsubscribe;
+}
+export interface PreviewStreamParams {
+    getCurrentDevices: () => MediaDeviceInfoLike[];
+    getCurrentMedia: () => Media | undefined;
+    updateMainStream: (request: MediaDeviceRequest) => Promise<void>;
+    mediaSignal: MediaSignals['onMediaChanged'];
+    onEnded?: () => void;
+    fftSize?: number;
+    queueOptions?: Partial<AsyncQueueOptions>;
+    processors: MediaProcessor[];
+}
+export interface PreviewControllerProps {
+    media: Media;
+    audioInput?: MediaDeviceInfoLike;
+    videoInput?: MediaDeviceInfoLike;
+    updatingPreview: boolean;
+    updatingMain: boolean;
+    originalMainAudioInput?: MediaDeviceInfoLike;
+    discardMedia: boolean;
+    initialized: boolean;
+}
+export interface PreviewStreamController {
+    media: Media;
+    audioInputChanged: boolean;
+    videoInputChanged: boolean;
+    inputChanged: boolean;
+    audioInput: PreviewInput;
+    videoInput: PreviewInput;
+    updatingPreview: boolean;
+    updatingMain: boolean;
+    updateAudioInput(input: PreviewInput): void;
+    updateVideoInput(input: PreviewInput): void;
+    applyChanges(force?: boolean): Promise<void>;
+    revertChanges(): Promise<void>;
+    onMediaChanged(callback: EventCallback<Media>): Unsubscribe;
+    onAudioInputChanged(callback: EventCallback<PreviewInput>): Unsubscribe;
+    onVideoInputChanged(callback: EventCallback<PreviewInput>): Unsubscribe;
+    onUpdatingPreview(callback: EventCallback<boolean>): Unsubscribe;
+    onUpdatingMain(callback: EventCallback<boolean>): Unsubscribe;
+    onAudioInputError(callback: EventErrorCallback): Unsubscribe;
+    onVideoInputError(callback: EventErrorCallback): Unsubscribe;
+    onApplyChangesError(callback: EventErrorCallback): Unsubscribe;
+    onRevertChangesError(callback: EventErrorCallback): Unsubscribe;
+}
+export declare const createPreviewStreamController: ({ getCurrentDevices, getCurrentMedia, updateMainStream, onEnded, mediaSignal, queueOptions, processors, }: PreviewStreamParams) => PreviewStreamController;
+export type CreatePreviewStreamController = typeof createPreviewStreamController;
+export {};