@pexip/media 17.2.0 → 17.4.0

package/dist/typeGuard.js

@@ -0,0 +1,40 @@
+import { isAudioNodeInit } from '@pexip/media-processor';
+import { UserMediaStatus, AUDIO_CONTENT_HINTS, VIDEO_CONTENT_HINTS, } from './types';
+export const isNonNullObject = (value) => {
+    if (typeof value === 'object' && value !== null) {
+        return true;
+    }
+    return false;
+};
+export const isUserMediaStatus = (value) => {
+    if (value && typeof value === 'string') {
+        return Object.values(UserMediaStatus).includes(value);
+    }
+    return false;
+};
+export const isMedia = (value) => {
+    if (isNonNullObject(value) && 'release' in value) {
+        return true;
+    }
+    return false;
+};
+export const isAnalyzerNodeInitProp = (value) => {
+    if (value === undefined || isAudioNodeInit(value)) {
+        return true;
+    }
+    return false;
+};
+export const isAudioContentHint = (value) => {
+    if (typeof value === 'string' &&
+        Object.values(AUDIO_CONTENT_HINTS).includes(value)) {
+        return true;
+    }
+    return false;
+};
+export const isVideoContentHint = (value) => {
+    if (typeof value === 'string' &&
+        Object.values(VIDEO_CONTENT_HINTS).includes(value)) {
+        return true;
+    }
+    return false;
+};

package/dist/types.d.ts

@@ -0,0 +1,559 @@
+import type { InputConstraintSet, MediaDeviceInfoLike, MediaDeviceRequest } from '@pexip/media-control';
+import type { RenderParams, RenderEffects, Segmenter, SegmentationModel } from '@pexip/media-processor';
+import type { Signal } from '@pexip/signal';
+export type Unsubscribe = () => void;
+export interface ExtendedMediaTrackSettings extends MediaTrackSettings {
+    mixWithAdditionalMedia?: boolean;
+    channelCount?: number;
+    denoise?: boolean;
+    vad?: boolean;
+    asd?: boolean;
+    backgroundBlurAmount?: number;
+    bgImageUrl?: string;
+    edgeBlurAmount?: number;
+    flipHorizontal?: boolean;
+    foregroundThreshold?: number;
+    videoSegmentation?: RenderEffects;
+    videoSegmentationModel?: SegmentationModel;
+    pan?: boolean;
+    tilt?: boolean;
+    zoom?: boolean;
+    contentHint?: AudioContentHint | VideoContentHint;
+}
+export type ExtendedMediaTrackSettingsKey = keyof ExtendedMediaTrackSettings;
+export interface MediaSettings {
+    audio: ExtendedMediaTrackSettings[];
+    video: ExtendedMediaTrackSettings[];
+}
+/**
+ * URL links needed for creating a denoise node for audio processing
+ */
+export interface DenoiseParams {
+    /**
+     * wasm URL to get the denoise wasm
+     */
+    wasmURL: string;
+    /**
+     * AudioWorklet module URL to get the worklet, @see AudioContext['audioWorklet']['addModule']
+     */
+    workletModule: string;
+    /**
+     * AudioWorklet options to pass @see AudioContext['audioWorklet']['addModule']
+     */
+    workletOptions?: WorkletOptions;
+}
+export interface MediaAttributes {
+    /**
+     * The constraints used to request the media
+     */
+    constraints?: MediaDeviceRequest;
+    /**
+     * The devices used for the media
+     */
+    devices: MediaDeviceInfoLike[];
+    /**
+     * Media stream for the media
+     */
+    stream?: MediaStream;
+    /**
+     * The raw stream obtained from the `getUserMedia` API
+     */
+    rawStream?: MediaStream;
+    /**
+     * Audio input device is used for the audio track from the current stream
+     */
+    audioInput?: MediaDeviceInfoLike;
+    /**
+     * Video input device is used for the video track from the current stream
+     */
+    videoInput?: MediaDeviceInfoLike;
+    /**
+     * The audio input device which is expected to be used for the current
+     * stream based on the provided constraints
+     */
+    expectedAudioInput?: MediaDeviceInfoLike;
+    /**
+     * The video input device which is expected to be used for the current
+     * stream based on the provided constraints
+     */
+    expectedVideoInput?: MediaDeviceInfoLike;
+    /**
+     * The status of the media
+     */
+    status: UserMediaStatus;
+    /**
+     * Current mute state of audio track
+     * `undefined` means there is no such track from the stream
+     */
+    audioMuted: boolean | undefined;
+    /**
+     * Current mute state of video track
+     * `undefined` means there is no such track from the stream
+     */
+    videoMuted: boolean | undefined;
+}
+export interface Media extends MediaAttributes {
+    /**
+     * mute/unmute the audio track
+     */
+    muteAudio(mute: boolean): void;
+    /**
+     * mute/unmute the video track
+     */
+    muteVideo(mute: boolean): void;
+    /**
+     * Apply the constraints to the current media
+     */
+    applyConstraints(constraints: MediaDeviceRequest): Promise<void>;
+    /**
+     * Release the media resources, e.g. camera/microphone
+     */
+    release(): Promise<void>;
+    getSettings(): MediaSettings;
+    toJSON?: () => unknown;
+}
+export type Process<T> = (a: T) => Promise<Media>;
+export type MediaProcessor = Process<Promise<Media>>;
+/**
+ * A media pipeline to get and process media
+ */
+export type Pipeline<T = MediaDeviceRequest> = [
+    Process<T>,
+    ...MediaProcessor[]
+];
+export type ProcessMedia = (media: Media) => undefined | Media;
+/**
+ * Use which processor API to process the stream track
+ * `stream` - Use MediaStreamTrackProcessor, when available
+ * `canvas` - Use Canvas
+ */
+export type VideoStreamTrackProcessorAPIs = 'stream' | 'canvas';
+export declare enum UserMediaStatus {
+    /**
+     * The initial status
+     */
+    Initial = "initial",
+    /**
+     * When we know the permissions were already granted from permission API
+     */
+    InitialPermissionsGranted = "initial-permissions-granted",
+    /**
+     * When we do not know the permissions from permission API
+     */
+    InitialPermissionsNotGranted = "initial-permissions-not-granted",
+    /**
+     * When video input permissions are initially denied in the browser and audio input permissions are unknown
+     */
+    InitialPermissionsVideoInputDenied = "initial-permissions-videoinput-denied",
+    /**
+     * When audio input permissions are initially denied in the browser and video input permissions are unknown
+     */
+    InitialPermissionsAudioInputDenied = "initial-permissions-audioinput-denied",
+    /**
+     * When video input permissions are granted in the browser and audio input permissions are unknown
+     */
+    InitialPermissionsVideoInputGranted = "initial-permissions-videoinput-granted",
+    /**
+     * When audio input permissions are granted in the browser and video input permissions are unknown
+     */
+    InitialPermissionsAudioInputGranted = "initial-permissions-audioinput-granted",
+    /**
+     * When audio input permissions are granted but video input permissions are initially denied
+     */
+    InitialPermissionsGrantedVideoInputDenied = "initial-permissions-audioinput-granted-videoinput-denied",
+    /**
+     * When video input permissions are granted but audio input permissions are initially denied
+     */
+    InitialPermissionsGrantedAudioInputDenied = "initial-permissions-videoinput-granted-audioinput-denied",
+    /**
+     * When there is no any kind of input devices
+     */
+    NoDevicesFound = "no-devices-found",
+    /**
+     * When there is no any video input devices
+     */
+    NoVideoDevicesFound = "no-video-devices-found",
+    /**
+     * When there is no any audio input devices
+     */
+    NoAudioDevicesFound = "no-audio-devices-found",
+    /**
+     * Derived from `MediaDeviceFailure.AudioInputDeviceNotFoundError`
+     */
+    AudioDeviceNotFound = "audio-device-not-found",
+    /**
+     * Derived from `MediaDeviceFailure.VideoInputDeviceNotFoundError`
+     */
+    VideoDeviceNotFound = "video-device-not-found",
+    /**
+     * Derived from `MediaDeviceFailure.AudioAndVideoDeviceNotFoundError`
+     */
+    AudioVideoDevicesNotFound = "audio-video-devices-not-found",
+    /**
+     * When Permission is granted by user for both video and audio, and both
+     * devices are exactly matched with the request constraints
+     */
+    PermissionsGranted = "permissions-granted",
+    /**
+     * When Permission is granted by user for both video and audio, and both
+     * devices are NOT exactly matched with the request constraints
+     */
+    PermissionsGrantedFallback = "permissions-granted-fallback-devices",
+    /**
+     * When Permission is granted by user for both video and audio, and audio
+     * input is NOT exactly matched with the request constraints
+     */
+    PermissionsGrantedFallbackAudioinput = "permissions-granted-fallback-audioinput",
+    /**
+     * When Permission is granted by user for both video and audio, and video
+     * input is NOT exactly matched with the request constraints
+     */
+    PermissionsGrantedFallbackVideoinput = "permissions-granted-fallback-videoinput",
+    /**
+     * When Permission for using both audio and video devices are rejected by the user
+     * from `PermissionDeniedError`
+     */
+    PermissionsRejected = "permissions-rejected",
+    /**
+     * When Permission for using the audio device is rejected by the user
+     * from `PermissionDeniedError`
+     */
+    PermissionsRejectedAudioInput = "permissions-rejected-audioinput",
+    /**
+     * When Permission for using the video device is rejected by the user
+     * from `PermissionDeniedError`
+     */
+    PermissionsRejectedVideoInput = "permissions-rejected-videoinput",
+    /**
+     * When only request and return exact audio input device
+     */
+    PermissionsOnlyAudioinput = "permissions-only-audioinput",
+    /**
+     * When only request audio input device because of no video devices
+     * available and returned exact audio input device
+     */
+    PermissionsOnlyAudioinputNoVideoDevices = "permissions-only-audioinput-no-video-devices",
+    /**
+     * When only request and returned NOT exact audio input device
+     */
+    PermissionsOnlyAudioinputFallback = "permissions-only-fallback-audioinput",
+    /**
+     * When only request audio input device because of no video devices
+     * available and returned NOT exact audio input device
+     */
+    PermissionsOnlyAudioinputFallbackNoVideoDevices = "permissions-only-fallback-audioinput-no-video-devices",
+    /**
+     * When only request and return exact video input device
+     */
+    PermissionsOnlyVideoinput = "permissions-only-videoinput",
+    /**
+     * When only request video input device because of no audio devices
+     * available and returned exact video input device
+     */
+    PermissionsOnlyVideoinputNoAudioDevices = "permissions-only-videoinput-no-audio-devices",
+    /**
+     * When only request and returned NOT exact video input device
+     */
+    PermissionsOnlyVideoinputFallback = "permissions-only-fallback-videoinput",
+    /**
+     * When only request video input device because of no video devices
+     * available and returned NOT exact video input device
+     */
+    PermissionsOnlyVideoinputFallbackNoAudioDevices = "permissions-only-fallback-videoinput-no-audio-devices",
+    /**
+     * When requesting the audio device is used by other application
+     */
+    AudioDeviceInUse = "audio-device-in-use",
+    /**
+     * When requesting the video device is used by other application
+     */
+    VideoDeviceInUse = "video-device-in-use",
+    /**
+     * When requesting both audio and video devices are used by other application
+     */
+    DevicesInUse = "devices-in-use",
+    /**
+     * When requesting both audio and video with over-constrained
+     */
+    Overconstrained = "overconstrained",
+    /**
+     * When requesting video with over-constrained
+     */
+    VideoOverconstrained = "video-overconstrained",
+    /**
+     * When requesting audio with over-constrained
+     */
+    AudioOverconstrained = "audio-overconstrained",
+    /**
+     * When requesting both audio and video with invalid constraints
+     */
+    InvalidConstraints = "invalid-constraints",
+    /**
+     * When requesting video with invalid constraints
+     */
+    InvalidVideoConstraints = "invalid-video-constraints",
+    /**
+     * When requesting audio with invalid constraints
+     */
+    InvalidAudioConstraints = "invalid-audio-constraints",
+    /**
+     * When requesting both audio and video with NotSupportedError
+     */
+    NotSupportedError = "not-supported-error",
+    /**
+     * When requesting video with NotSupportedError
+     */
+    NotSupportedErrorOnlyVideoInput = "not-supported-error-only-video",
+    /**
+     * When requesting audio with NotSupportedError
+     */
+    NotSupportedErrorOnlyAudioInput = "not-supported-error-only-audio",
+    /**
+     * Unknown error from both video and audio
+     */
+    UnknownError = "unknown-error",
+    /**
+     * Unknown error from audio device
+     */
+    UnknownErrorOnlyAudioinput = "unknown-error-only-audioinput",
+    /**
+     * Unknown error from video device
+     */
+    UnknownErrorOnlyVideoinput = "unknown-error-only-videoinput"
+}
+export interface AudioSignalDetectionOptions {
+    /**
+     * Audio Signal Detection Duration in second
+     */
+    audioSignalDetectionDuration?: number;
+    /**
+     * Whether or not to detect audio signal for malfunctioning device
+     */
+    shouldDetectAudio: () => boolean;
+}
+export interface VoiceActivityDetectionOptions {
+    /**
+     * Voice Activity Detection in millisecond
+     */
+    vadThrottleMS?: number;
+    /**
+     * Whether or not to detect voice activity
+     */
+    shouldDetectVoiceActivity: () => boolean;
+}
+interface DeviceMuteState {
+    /**
+     * Indicating whether audio is muted
+     */
+    audio: boolean;
+    /**
+     * Indicating whether video is muted
+     */
+    video: boolean;
+}
+export interface MediaOptions {
+    /**
+     * Media signals to be used for the module
+     *
+     * @see MediaSignals
+     */
+    signals: MediaSignals;
+    /**
+     * Media Processors
+     */
+    mediaProcessors: MediaProcessor[];
+    /**
+     * A function to get the devices' mute state
+     */
+    getMuteState: () => DeviceMuteState;
+    /**
+     * Pass default constraints to use with get media wrappers
+     */
+    getDefaultConstraints?: () => {
+        audio?: InputConstraintSet | false;
+        video?: InputConstraintSet | false;
+    };
+}
+export interface MediaProps {
+    /**
+     * Current media
+     */
+    media: Media;
+    /**
+     * Current device
+     */
+    devices: MediaDeviceInfoLike[];
+    /**
+     * When should we discard the requested MediaStream
+     */
+    discardMedia: boolean;
+}
+export interface MediaController {
+    /**
+     * Current Media
+     */
+    media: Media;
+    /**
+     * Current device list
+     */
+    devices: MediaDeviceInfoLike[];
+    /**
+     * Execute the media pipeline immediately with provided constraints
+     *
+     * @param constraints - @see MediaDeviceRequest
+     */
+    getUserMedia: (constraints: MediaDeviceRequest) => void;
+    /**
+     * Execute the media pipeline immediately with provided constraints. An
+     * explicit version of `getUserMedia` to make it possible to chain other
+     * async actions
+     *
+     * @param constraints - @see MediaDeviceRequest
+     */
+    getUserMediaAsync: (constraints: MediaDeviceRequest) => Promise<void>;
+    /**
+     * Cross-check PermissionState and available devices before requesting user
+     * media, the request will be skipped if there is no granted device.
+     */
+    tryAndGetUserMedia: (constraints: MediaDeviceRequest) => void;
+}
+export type VideoRenderParams = Omit<RenderParams, 'backgroundImage' | 'effects'> & {
+    /**
+     * Target frame rate for the video segmentation
+     */
+    frameRate: number;
+    /**
+     * Default background image URL for overlay effects
+     */
+    bgImageUrl: string;
+    /**
+     * Render Effects
+     */
+    videoSegmentation: RenderEffects;
+    pan?: boolean;
+    tilt?: boolean;
+    zoom?: boolean;
+};
+export interface StreamTrackSignals {
+    /**
+     * MediaStreamTrack events: mute
+     * https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
+     */
+    onStreamTrackMuted: Signal<MediaStreamTrack>;
+    /**
+     * MediaStreamTrack events: unmute
+     * https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
+     */
+    onStreamTrackUnmuted: Signal<MediaStreamTrack>;
+    /**
+     * MediaStreamTrack events: ended
+     * https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
+     */
+    onStreamTrackEnded: Signal<MediaStreamTrack>;
+    /**
+     * Emit `MediaStreamTrack` whenever a call to `Media['muteAudio']` or
+     * `Media['muteVideo']`
+     */
+    onStreamTrackEnabled: Signal<MediaStreamTrack>;
+}
+export interface MediaChangesSignals {
+    onDevicesChanged: Signal<MediaDeviceInfoLike[]>;
+    onStatusChanged: Signal<UserMediaStatus>;
+    onMediaChanged: Signal<Media>;
+}
+export interface AudioDetectionSignals {
+    onVAD: Signal<undefined>;
+    onSilentDetected: Signal<boolean>;
+}
+export type MediaSignalsOptional = Pick<Partial<MediaChangesSignals>, 'onDevicesChanged' | 'onStatusChanged'> & Partial<StreamTrackSignals>;
+export type MediaSignalsRequired = Pick<MediaChangesSignals, 'onMediaChanged'> & AudioDetectionSignals;
+export type MediaSignals = MediaSignalsRequired & MediaSignalsOptional;
+export declare enum DeniedDevices {
+    Microphone = "microphone",
+    Camera = "camera",
+    Both = "microphone-and-camera"
+}
+export interface Segmenters {
+    mediapipeSelfie: Segmenter;
+}
+/**
+ * Audio content hints are only applicable when the MediaStreamTrack contains an
+ * audio track
+ *
+ * {@link https://www.w3.org/TR/mst-content-hint/#audio-content-hints}
+ */
+export declare const AUDIO_CONTENT_HINTS: {
+    /**
+     * No hint has been provided, the implementation should make its
+     * best-informed guess on how to handle contained audio data. This may be
+     * inferred from how the track was opened or by doing content analysis
+     */
+    readonly NoHint: "";
+    /**
+     * The track should be treated as if it contains speech data. Consuming this
+     * signal it may be appropriate to apply noise suppression or boost
+     * intelligibility of the incoming signal.
+     */
+    readonly Speech: "speech";
+    /**
+     * The track should be treated as if it contains data for the purpose of
+     * speech recognition by a machine. Consuming this signal it may be
+     * appropriate to boost intelligibility of the incoming signal for
+     * transcription and turn off audio-processing components that are used for
+     * human consumption.
+     */
+    readonly SpeechRecognition: "speech-recognition";
+    /**
+     * The track should be treated as if it contains music data. Generally this
+     * might imply tuning or turning off audio-processing components that are
+     * used to process speech data to prevent the audio from being distorted.
+     */
+    readonly Music: "music";
+};
+export type AudioContentHint = (typeof AUDIO_CONTENT_HINTS)[keyof typeof AUDIO_CONTENT_HINTS];
+/**
+ * Video content hints are only applicable when the MediaStreamTrack contains a
+ * video track.
+ *
+ * {@link https://www.w3.org/TR/mst-content-hint/#video-content-hints}
+ */
+export declare const VIDEO_CONTENT_HINTS: {
+    /**
+     * No hint has been provided, the implementation should make its
+     * best-informed guess on how contained video content should be treated.
+     * This can for example be inferred from how the track was opened or by
+     * doing content analysis.
+     */
+    readonly NoHint: "";
+    /**
+     * The track should be treated as if it contains video where motion is
+     * important. This is normally webcam video, movies or video games.
+     * Quantization artefacts and downscaling are acceptable in order to
+     * preserve motion as well as possible while still retaining target
+     * bitrates. During low bitrates when compromises have to be made, more
+     * effort is spent on preserving frame rate than edge quality and details.
+     */
+    readonly Motion: "motion";
+    /**
+     * The track should be treated as if video details are extra important.
+     * This is generally applicable to presentations or web pages with text
+     * content, painting or line art. This setting would normally optimize for
+     * detail in the resulting individual frames rather than smooth playback.
+     * Artefacts from quantization or downscaling that make small text or line
+     * art unintelligible should be avoided.
+     */
+    readonly Detail: "detail";
+    /**
+     * The track should be treated as if video details are extra important, and
+     * that significant sharp edges and areas of consistent color can occur
+     * frequently. This is generally applicable to presentations or web pages
+     * with text content. This setting would normally optimize for detail in the
+     * resulting individual frames rather than smooth playback, and may take
+     * advantage of encoder tools that optimize for text rendering. Artefacts
+     * from quantization or downscaling that make small text or line art
+     * unintelligible should be avoided.
+     */
+    readonly Text: "text";
+};
+export type VideoContentHint = (typeof VIDEO_CONTENT_HINTS)[keyof typeof VIDEO_CONTENT_HINTS];
+export {};