@pexip/media 17.3.0 → 18.0.0

package/dist/utils.d.ts

@@ -0,0 +1,108 @@
+import type { MediaDeviceRequest, MediaDeviceInfoLike } from '@pexip/media-control';
+import { UserMediaStatus } from './types';
+import type { Media, MediaAttributes, Pipeline, Process, ProcessMedia, ExtendedMediaTrackSettingsKey, ExtendedMediaTrackSettings } from './types';
+export declare const makeDeriveDeviceStatus: (constraints: MediaDeviceRequest) => (audio: UserMediaStatus, video: UserMediaStatus, both: UserMediaStatus) => UserMediaStatus;
+export declare const createMediaProcess: (process: ProcessMedia) => Process<Promise<Media>>;
+export declare const createMediaPipeline: <T = MediaDeviceRequest>(init: Pipeline<T> | (() => Pipeline<T>)) => {
+    pipe: (process: Process<Promise<Media>>) => void;
+    execute: (m: T) => Promise<Media>;
+};
+/**
+ * Interpret provided input to resolve to a MediaDeviceInfoLike when possible
+ * otherwise `undefined`
+ */
+export declare const interpretInput: (input: boolean | MediaDeviceInfoLike | undefined, getCurrentInput: () => MediaDeviceInfoLike | undefined) => MediaDeviceInfoLike | undefined;
+type InputConstraints = MediaDeviceRequest['audio'];
+interface MediaInputInfo {
+    devices: MediaDeviceInfoLike[];
+    input: MediaDeviceInfoLike | undefined;
+}
+/**
+ * Memorized Expected Input
+ */
+export declare const createMemorizedGetExpectedInput: () => (constraints: InputConstraints, getInfo: () => MediaInputInfo) => ExpectedInput;
+/**
+ * A utility function to check if the provided track is muted. There are
+ * 2 factors to be considered: `MediaStreamTrack['muted']` and `MediaStreamTrack['enabled']`.
+ *
+ * ```
+ * | muted \ enabled | true  | false |
+ * |-----------------| ----- | ----- |
+ * |     true        | true  | true  |
+ * |     false       | false | true  |
+ * ```
+ *
+ * @param tracks - The tracks can be got from `MediaStream['getAudioStats']` or
+ * `MediaStream['getVideoTracks']`
+ *
+ * @returns `true` means muted, `false` means not muted and `undefined` means
+ * there is no track to check
+ */
+export declare const isMuted: (tracks: MediaStreamTrack[] | undefined) => boolean | undefined;
+type ExpectedInput = MediaDeviceInfoLike | undefined;
+export declare const buildMedia: (getMedia: () => Partial<Media>, onSetStatus?: ((status: UserMediaStatus) => void) | undefined) => Media;
+/**
+ * Clone the media from the rawStream (if any), otherwise, stream
+ */
+export declare const cloneMedia: (media: Media) => Promise<Media>;
+/**
+ * Shallow copy the provided object and override with provided overriding
+ *
+ * @param original - Original object
+ * @param overriding - Object of the same type to override the original
+ *
+ * @returns a shallow copied object
+ */
+export declare const shallowCopy: <T>(original: T, overriding: Partial<T>) => T;
+export declare const getDevicesChanges: (prev: MediaDeviceInfoLike[], next: MediaDeviceInfoLike[]) => {
+    unauthorized: MediaDeviceInfoLike[];
+    authorized: MediaDeviceInfoLike[];
+    found: MediaDeviceInfoLike[];
+    lost: MediaDeviceInfoLike[];
+    devices: MediaDeviceInfoLike[];
+};
+/**
+ * Apply Extended constraints on top of the original
+ *
+ * @param media - The media from the media pipeline
+ * @param applyExtended - The function to be called when the previous
+ * `applyConstraints` is done
+ */
+export declare const applyExtendedConstraints: (media: Media, applyExtended: (constraints: MediaDeviceRequest) => Promise<void>) => (constraints: MediaDeviceRequest) => Promise<void>;
+export declare const AUDIO_SETTINGS_KEYS: ExtendedMediaTrackSettingsKey[];
+export declare const VIDEO_SETTINGS_KEYS: ExtendedMediaTrackSettingsKey[];
+export declare const MIXING_SETTINGS_KEYS: ExtendedMediaTrackSettingsKey[];
+export declare const hasSettingsChanged: (keysToLookFor: ExtendedMediaTrackSettingsKey[]) => (settingsA: ExtendedMediaTrackSettings | undefined, settingsB: ExtendedMediaTrackSettings | undefined) => boolean;
+export declare const toJSON: (media: Partial<MediaAttributes>) => {
+    constraints: MediaDeviceRequest | undefined;
+    devices: MediaDeviceInfoLike[] | undefined;
+    stream: MediaStream | undefined;
+    rawStream: MediaStream | undefined;
+    audioInput: MediaDeviceInfoLike | undefined;
+    videoInput: MediaDeviceInfoLike | undefined;
+    expectedAudioInput: MediaDeviceInfoLike | undefined;
+    expectedVideoInput: MediaDeviceInfoLike | undefined;
+    status: UserMediaStatus | undefined;
+    audioMuted: boolean | undefined;
+    videoMuted: boolean | undefined;
+};
+export declare const wrapToJSON: (media: Media) => Media;
+/**
+ * A function to get the blur kernel size of image height
+ *
+ * @param percentage - The percentage of image height to calculate the blur
+ * kernel size
+ * @param height - The image height
+ * @param max - The upper bound
+ *
+ * @returns blur kernel size
+ */
+export declare const getBlurKernelSize: (percentage: number, height: number, max?: number) => number;
+/**
+ * Apply the content hint to the track
+ *
+ * @param hint - Content hint
+ * @param track - The track to be applied
+ */
+export declare const applyContentHint: <T extends "" | "speech" | "speech-recognition" | "music" | "motion" | "detail" | "text">(hint?: T | undefined) => (track: MediaStreamTrack) => void;
+export {};

package/dist/utils.js

@@ -0,0 +1,361 @@
+import { applyConstraints, createTrackDevicesChanges, extractConstraintsWithKeys, findDeviceFromConstraints, muteStreamTrack, relaxInputConstraint, stopMediaStream, isAudioInput, isVideoInput, } from '@pexip/media-control';
+import { calculateMaxBlurPass } from '@pexip/media-processor';
+import { UserMediaStatus } from './types';
+import { isOverConstrained } from './status';
+import { isMedia } from './typeGuard';
+export const makeDeriveDeviceStatus = (constraints) => (audio, video, both) => {
+    if (!constraints.audio && constraints.video) {
+        return video;
+    }
+    if (!constraints.video && constraints.audio) {
+        return audio;
+    }
+    return both;
+};
+export const createMediaProcess = (process) => async (mediaP) => {
+    const media = await mediaP;
+    return process(media) ?? media;
+};
+export const createMediaPipeline = (init) => {
+    const getPipeline = () => (typeof init === 'function' ? init() : init);
+    return {
+        pipe: (process) => {
+            getPipeline().push(process);
+        },
+        execute: async (m) => {
+            const [first, ...processes] = getPipeline();
+            if (first) {
+                const piped = processes.reduce((prev, next) => next(prev), first(m));
+                return piped;
+            }
+            const media = m instanceof Promise ? (await m) : m;
+            if (isMedia(media)) {
+                return Promise.resolve(media);
+            }
+            throw new Error('Expect a media input or a processor');
+        },
+    };
+};
+/**
+ * Interpret provided input to resolve to a MediaDeviceInfoLike when possible
+ * otherwise `undefined`
+ */
+export const interpretInput = (input, getCurrentInput) => {
+    if (input === true || input === undefined) {
+        return getCurrentInput();
+    }
+    if (input === false) {
+        return undefined;
+    }
+    return input;
+};
+/**
+ * Memorized Expected Input
+ */
+export const createMemorizedGetExpectedInput = () => {
+    const props = {
+        cachedExpectedInputs: new Map(),
+    };
+    return (constraints, getInfo) => {
+        if (props.cachedExpectedInputs.has(constraints)) {
+            return props.cachedExpectedInputs.get(constraints);
+        }
+        const { devices, input } = getInfo();
+        // Update cache
+        const relaxedConstraints = relaxInputConstraint(constraints, devices);
+        const { device: [[device] = []], } = extractConstraintsWithKeys(['device'])(relaxedConstraints);
+        // The result from `findDeviceFromConstraints` has more restrictive
+        // result since it also consider if the device can be found from the
+        // device list
+        const found = device ?? findDeviceFromConstraints(constraints, devices);
+        const expectedInput = interpretInput(found, () => input);
+        props.cachedExpectedInputs.clear();
+        props.cachedExpectedInputs.set(constraints, expectedInput);
+        return expectedInput;
+    };
+};
+/**
+ * A utility function to check if the provided track is muted. There are
+ * 2 factors to be considered: `MediaStreamTrack['muted']` and `MediaStreamTrack['enabled']`.
+ *
+ * ```
+ * | muted \ enabled | true  | false |
+ * |-----------------| ----- | ----- |
+ * |     true        | true  | true  |
+ * |     false       | false | true  |
+ * ```
+ *
+ * @param tracks - The tracks can be got from `MediaStream['getAudioStats']` or
+ * `MediaStream['getVideoTracks']`
+ *
+ * @returns `true` means muted, `false` means not muted and `undefined` means
+ * there is no track to check
+ */
+export const isMuted = (tracks) => {
+    if (!tracks?.length) {
+        return undefined;
+    }
+    return !tracks.some(track => !track.muted && track.enabled);
+};
+export const buildMedia = (getMedia, onSetStatus) => {
+    const props = {
+        status: getMedia().status ?? UserMediaStatus.Initial,
+        devices: getMedia().devices ?? [],
+        constraints: getMedia().constraints,
+    };
+    const getExpectedAudioInput = createMemorizedGetExpectedInput();
+    const getExpectedVideoInput = createMemorizedGetExpectedInput();
+    const muteTrack = (kind) => (muted) => {
+        const { muteAudio, muteVideo, stream } = getMedia();
+        const mute = kind === 'audio' ? muteAudio : muteVideo;
+        if (mute) {
+            return mute(muted);
+        }
+        return muteStreamTrack(stream)(muted, kind);
+    };
+    const release = () => {
+        const { release, stream } = getMedia();
+        if (release) {
+            return release();
+        }
+        return new Promise(resolve => {
+            stopMediaStream(stream);
+            resolve();
+        });
+    };
+    return {
+        get constraints() {
+            return props.constraints;
+        },
+        get devices() {
+            return props.devices;
+        },
+        set devices(newDevices) {
+            props.devices = newDevices;
+        },
+        get stream() {
+            return getMedia().stream;
+        },
+        get expectedAudioInput() {
+            const media = getMedia();
+            return getExpectedAudioInput(props.constraints?.audio, () => ({
+                devices: media.devices?.filter(isAudioInput) ?? [],
+                input: media.audioInput,
+            }));
+        },
+        get expectedVideoInput() {
+            const media = getMedia();
+            return getExpectedVideoInput(props.constraints?.video, () => ({
+                devices: media.devices?.filter(isVideoInput) ?? [],
+                input: media.videoInput,
+            }));
+        },
+        get rawStream() {
+            const { rawStream, stream } = getMedia();
+            return rawStream ?? stream;
+        },
+        get audioInput() {
+            return getMedia().audioInput;
+        },
+        get videoInput() {
+            return getMedia().videoInput;
+        },
+        get status() {
+            return props.status;
+        },
+        set status(status) {
+            props.status = status;
+            onSetStatus?.(status);
+        },
+        set constraints(value) {
+            props.constraints = value;
+        },
+        get audioMuted() {
+            return isMuted(getMedia().stream?.getAudioTracks());
+        },
+        get videoMuted() {
+            return isMuted(getMedia().stream?.getVideoTracks());
+        },
+        muteAudio: muteTrack('audio'),
+        muteVideo: muteTrack('video'),
+        applyConstraints: async (constraints) => {
+            const { stream, applyConstraints: prevApplyConstraints } = getMedia();
+            if (prevApplyConstraints) {
+                return await prevApplyConstraints(constraints);
+            }
+            return await applyConstraints(stream?.getTracks(), constraints);
+        },
+        release,
+        getSettings: () => {
+            const { getSettings, stream } = getMedia();
+            if (!stream) {
+                return {
+                    audio: [],
+                    video: [],
+                };
+            }
+            if (getSettings) {
+                return getSettings();
+            }
+            return {
+                audio: stream
+                    .getAudioTracks()
+                    .map(track => track.getSettings()),
+                video: stream
+                    .getVideoTracks()
+                    .map(track => track.getSettings()),
+            };
+        },
+        toJSON: () => toJSON(getMedia()),
+    };
+};
+/**
+ * Clone the media from the rawStream (if any), otherwise, stream
+ */
+export const cloneMedia = async (media) => {
+    const stream = (media.rawStream ?? media.stream)?.clone();
+    // Restore the enabled state for all cloned track
+    stream?.getTracks().forEach(track => (track.enabled = true));
+    const { audio, video } = media.getSettings();
+    const clonedMedia = buildMedia(() => ({
+        stream,
+        constraints: media?.constraints,
+        devices: media?.devices,
+        status: media?.status,
+        rawStream: stream,
+        audioInput: media?.audioInput,
+        videoInput: media?.videoInput,
+        getSettings: () => ({ audio, video }),
+    }));
+    return Promise.resolve(clonedMedia);
+};
+/**
+ * Shallow copy the provided object and override with provided overriding
+ *
+ * @param original - Original object
+ * @param overriding - Object of the same type to override the original
+ *
+ * @returns a shallow copied object
+ */
+export const shallowCopy = (original, overriding) => {
+    const copy = Object.create(Object.getPrototypeOf(original), Object.getOwnPropertyDescriptors(original));
+    return Object.defineProperties(copy, Object.getOwnPropertyDescriptors(overriding));
+};
+export const getDevicesChanges = (prev, next) => {
+    const trackChanges = createTrackDevicesChanges(prev);
+    return trackChanges(next);
+};
+/**
+ * Apply Extended constraints on top of the original
+ *
+ * @param media - The media from the media pipeline
+ * @param applyExtended - The function to be called when the previous
+ * `applyConstraints` is done
+ */
+export const applyExtendedConstraints = (media, applyExtended) => 
+/**
+ * Apply constraints
+ * @param constraints - The constraints to be applied to the media
+ */
+async (constraints) => {
+    await media.applyConstraints(constraints);
+    if (!isOverConstrained(media.status)) {
+        await applyExtended(constraints);
+    }
+};
+export const AUDIO_SETTINGS_KEYS = [
+    'denoise',
+    'vad',
+    'asd',
+    'contentHint',
+];
+export const VIDEO_SETTINGS_KEYS = [
+    'frameRate',
+    'videoSegmentation',
+    'videoSegmentationModel',
+    'foregroundThreshold',
+    'backgroundBlurAmount',
+    'edgeBlurAmount',
+    'maskCombineRatio',
+    'backgroundImageUrl',
+    'width',
+    'height',
+    'contentHint',
+];
+export const MIXING_SETTINGS_KEYS = [
+    'mixWithAdditionalMedia',
+];
+export const hasSettingsChanged = (keysToLookFor) => {
+    const cache = {};
+    return (settingsA, settingsB) => {
+        if (cache.result !== undefined &&
+            cache.settingsA === settingsA &&
+            cache.settingsB === settingsB) {
+            return cache.result;
+        }
+        cache.settingsA = settingsA;
+        cache.settingsB = settingsB;
+        for (const key of keysToLookFor) {
+            if (settingsA === settingsB) {
+                cache.result = false;
+                return cache.result;
+            }
+            if (settingsA === undefined || settingsB === undefined) {
+                cache.result = true;
+                return cache.result;
+            }
+            if (settingsA[key] !== settingsB[key]) {
+                cache.result = true;
+                return cache.result;
+            }
+        }
+        cache.result = false;
+        return cache.result;
+    };
+};
+export const toJSON = (media) => {
+    return {
+        constraints: media.constraints,
+        devices: media.devices,
+        stream: media.stream,
+        rawStream: media.rawStream,
+        audioInput: media.audioInput,
+        videoInput: media.videoInput,
+        expectedAudioInput: media.expectedAudioInput,
+        expectedVideoInput: media.expectedVideoInput,
+        status: media.status,
+        audioMuted: media.audioMuted,
+        videoMuted: media.videoMuted,
+    };
+};
+export const wrapToJSON = (media) => {
+    media.toJSON = () => toJSON(media);
+    return media;
+};
+/**
+ * A function to get the blur kernel size of image height
+ *
+ * @param percentage - The percentage of image height to calculate the blur
+ * kernel size
+ * @param height - The image height
+ * @param max - The upper bound
+ *
+ * @returns blur kernel size
+ */
+export const getBlurKernelSize = (percentage, height, max = calculateMaxBlurPass(height)) => {
+    if (height <= 0 || percentage <= 0 || max <= 0) {
+        return 0;
+    }
+    return Math.min(Math.ceil(percentage * 0.01 * max), max);
+};
+/**
+ * Apply the content hint to the track
+ *
+ * @param hint - Content hint
+ * @param track - The track to be applied
+ */
+export const applyContentHint = (hint) => (track) => {
+    if (hint !== undefined && hint !== track.contentHint) {
+        track.contentHint = hint;
+    }
+};

package/dist/videoProcessor.d.ts

@@ -0,0 +1,41 @@
+import type { VideoProcessor, SegmentationTransform, SegmentationModel } from '@pexip/media-processor';
+import type { MediaDeviceRequest } from '@pexip/media-control';
+import type { Process, Media, VideoRenderParams, Segmenters, VideoStreamTrackProcessorAPIs, VideoContentHint } from './types';
+interface ProcessorDeps {
+    videoProcessor?: () => VideoProcessor;
+    transformer?: SegmentationTransform;
+    segmenters: Partial<Segmenters>;
+    videoSegmentationModel?: SegmentationModel;
+}
+interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<ProcessorDeps, 'videoProcessor'> {
+    /**
+     * What API to use for processing the MediaStreamTrack
+     * `stream` - Use MediaStreamTrackProcessor, when available
+     * `canvas` - Use Canvas
+     */
+    trackProcessorAPI?: () => VideoStreamTrackProcessorAPIs;
+    /**
+     * Whether or to enable this processor
+     */
+    shouldEnable: () => boolean;
+    /**
+     * Callback when error occurs
+     */
+    onError?: (error: Error) => void;
+    processingWidth: number;
+    processingHeight: number;
+    hasInitializedDeps?: boolean;
+    width?: number;
+    height?: number;
+    scope?: string;
+}
+interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<ProcessorDeps> {
+    hasInitialized: boolean;
+    contentHint?: VideoContentHint;
+}
+declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUrl", "maskCombineRatio", "edgeBlurAmount", "foregroundThreshold", "frameRate", "videoSegmentation", "videoSegmentationModel", "width", "height", "pan", "tilt", "zoom", "contentHint"];
+type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
+type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
+export declare const updateFeatureProps: (constraints: MediaDeviceRequest['video'], props: FeatureProps) => FeatureProps;
+export declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, backgroundImageUrl, maskCombineRatio, edgeBlurAmount, scope, ...options }: VideoStreamProcessOptions) => Process<Promise<Media>>;
+export {};