@pexip/media 21.0.0 → 22.0.0

package/dist/previewController.js

@@ -51,7 +51,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
     throttleInMS: DEFAULT_QUEUE_THROTTLE_MS,
     delayInMS: DEFAULT_QUEUE_DELAY_MS,
     dropLast: DEFAULT_QUEUE_DROP_LAST,
-}, audioProcessors, videoProcessors, getDefaultConstraints = () => ({}), signals, }) => {
+}, audioProcessors, videoProcessors, getDefaultConstraints = () => ({}), signals, stopVideoTrackAsMute = () => false, }) => {
     const queue = createAsyncQueue(queueOptions);
     const eventHandlers = {};
     const internalProps = {
@@ -62,6 +62,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
         discardMedia: false,
         initialized: false,
         signals,
+        keepVideoMuted: false,
     };
     const logger = createModuleLogger({
         module: 'PreviewStreamController',
@@ -125,7 +126,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
                 eventHandlers.audioMuted?.(track.muted);
             }
             else {
-                eventHandlers.videoMuted?.(track.muted);
+                eventHandlers.videoMuted?.(props.keepVideoMuted || track.muted);
             }
         }),
         props.signals.onMediaTrackSuspended?.add(track => {
@@ -151,7 +152,9 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
                     break;
                 }
                 case 'video': {
-                    eventHandlers.videoMuted?.(isTrackMuted(track) || isTrackEnded(track));
+                    eventHandlers.videoMuted?.(props.keepVideoMuted ||
+                        isTrackMuted(track) ||
+                        isTrackEnded(track));
                     break;
                 }
             }
@@ -163,7 +166,9 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
                     break;
                 }
                 case 'video': {
-                    eventHandlers.videoMuted?.(isTrackMuted(track) || isTrackEnded(track));
+                    eventHandlers.videoMuted?.(props.keepVideoMuted ||
+                        isTrackMuted(track) ||
+                        isTrackEnded(track));
                     break;
                 }
             }
@@ -173,7 +178,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
         getUserMedia: requestUserMediaWithRetry(() => Promise.resolve(getCurrentDevices())),
         signals: props.signals,
         getCurrentDevices: () => Promise.resolve(getCurrentDevices()),
-        stopVideoTrackAsMute: () => false,
+        stopVideoTrackAsMute,
         updateMedia: constraints => updateMedia(constraints),
     });
     const mergeMediaConstraints = (constraints) => {
@@ -223,6 +228,14 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
         getCurrentDevices: () => Promise.resolve(getCurrentDevices()),
         shouldDiscardMedia: () => props.discardMedia,
         onMediaTracksChanged: (media, tracks) => {
+            if (props.keepVideoMuted) {
+                for (const track of tracks) {
+                    if (track.kind === 'videoinput') {
+                        track.mute(true);
+                    }
+                }
+                props.keepVideoMuted = false;
+            }
             props.media = media;
             queue.enqueue(async () => {
                 await processAndUpdateMedia(media, tracks, false);
@@ -328,14 +341,16 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
             props.updatingPreviewAudio = false;
         }
     };
-    const updateVideoInput = async (input) => {
+    const updateVideoInput = async (input, syncMuteState = true) => {
         try {
             props.videoInput = input;
             props.updatingPreviewVideo = true;
             if (input === undefined) {
                 return await releaseVideo();
             }
-            return await updatePreviewMedia({ video: { device: { exact: input } } });
+            props.keepVideoMuted = syncMuteState && !!props.media?.videoMuted;
+            await updatePreviewMedia({ video: { device: { exact: input } } });
+            return;
         }
         catch (error) {
             if (error instanceof Error) {
@@ -475,7 +490,7 @@ export const createPreviewStreamController = ({ getCurrentDevices, getCurrentMed
                     return updateAudioInput(input);
                 }
                 case 'videoinput': {
-                    return updateVideoInput(input);
+                    return updateVideoInput(input, false);
                 }
             }
         },

package/dist/typeGuard.d.ts

@@ -1,5 +1,5 @@
 import type { AnalyzerNodeInit } from '@pexip/media-processor';
-import type { AudioContentHint, Media, VideoContentHint, VideoStreamTrackProcessorAPIs } from './types';
+import type { AudioContentHint, Media, VideoContentHint } from './types';
 import { UserMediaStatus } from './types';
 export declare const isNonNullObject: (value: unknown) => value is Record<string, unknown>;
 export declare const isUserMediaStatus: (value: unknown) => value is UserMediaStatus;
@@ -7,4 +7,3 @@ export declare const isMedia: (value: unknown) => value is Media;
 export declare const isAnalyzerNodeInitProp: (value: unknown) => value is AnalyzerNodeInit | undefined;
 export declare const isAudioContentHint: (value: unknown) => value is AudioContentHint;
 export declare const isVideoContentHint: (value: unknown) => value is VideoContentHint;
-export declare const isVideoStreamTrackProcessorAPIs: (value: unknown) => value is VideoStreamTrackProcessorAPIs;

package/dist/typeGuard.js

@@ -38,6 +38,3 @@ export const isVideoContentHint = (value) => {
     }
     return false;
 };
-export const isVideoStreamTrackProcessorAPIs = (value) => {
-    return (typeof value === 'string' && (value === 'stream' || value === 'canvas'));
-};

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 import type { IndexedDevices, InputConstraintSet, InputDeviceConstraint, MediaDeviceInfoLike, MediaDeviceRequest } from '@pexip/media-control';
-import type { RendererOptions, RenderEffects, Segmenter, SegmentationModel } from '@pexip/media-processor';
+import type { RendererOptions, RenderEffects } from '@pexip/media-processor';
 import type { Signal } from '@pexip/signal';
 export type Unsubscribe = () => void;
 export interface ExtendedMediaTrackSettings extends MediaTrackSettings {
@@ -14,10 +14,12 @@ export interface ExtendedMediaTrackSettings extends MediaTrackSettings {
     maskCombineRatio?: number;
     foregroundThreshold?: number;
     videoSegmentation?: RenderEffects;
-    videoSegmentationModel?: SegmentationModel;
-    pan?: boolean;
-    tilt?: boolean;
-    zoom?: boolean;
+    panEnabled?: boolean;
+    tiltEnabled?: boolean;
+    zoomEnabled?: boolean;
+    pan?: number;
+    tilt?: number;
+    zoom?: number;
     contentHint?: AudioContentHint | VideoContentHint;
 }
 export type ExtendedMediaTrackSettingsKey = keyof ExtendedMediaTrackSettings;
@@ -230,12 +232,6 @@ export type GetUserMediaProcess = (request: {
     };
     currentMedia: Media | undefined;
 }) => Promise<Media>;
-/**
- * Use which processor API to process the stream track
- * `stream` - Use `MediaStreamTrackProcessor`, when available
- * `canvas` - Use Canvas
- */
-export type VideoStreamTrackProcessorAPIs = 'stream' | 'canvas';
 export declare enum UserMediaStatus {
     /**
      * The initial status
@@ -573,9 +569,6 @@ export declare enum DeniedDevices {
     Camera = "camera",
     Both = "microphone-and-camera"
 }
-export type Segmenters = {
-    [Property in SegmentationModel]: Segmenter;
-};
 /**
  * Audio content hints are only applicable when the MediaStreamTrack contains an
  * audio track

package/dist/utils.js

@@ -3,6 +3,7 @@ import { calculateMaxBlurPass, clamping, } from '@pexip/media-processor';
 import { hasOwn, assert, isEmpty } from '@pexip/utils';
 import { internalSignals } from './signals';
 import { MAX_TENT_BLUR_SIZE } from './constants';
+import { mapFeatureToSettings } from './videoProcessor';
 export const makeDeriveDeviceStatus = (constraints) => (audio, video, both) => {
     if (constraints.audio) {
         if (constraints.video) {
@@ -571,7 +572,6 @@ export const AUDIO_SETTINGS_KEYS = [
 export const VIDEO_SETTINGS_KEYS = [
     'frameRate',
     'videoSegmentation',
-    'videoSegmentationModel',
     'foregroundThreshold',
     'backgroundBlurAmount',
     'edgeBlurAmount',
@@ -629,10 +629,11 @@ export const getSettingsFromKeys = (keys, settings) => {
         return settings;
     }
     const result = {};
+    const mapped = mapFeatureToSettings(settings);
     for (const key of keys) {
-        if (settings[key] !== undefined) {
+        if (mapped[key] !== undefined) {
             // @ts-expect-error --- Type issue and need to be fixed when we have time
-            result[key] = settings[key];
+            result[key] = mapped[key];
         }
     }
     return result;

package/dist/videoProcessor.d.ts

@@ -1,27 +1,15 @@
-import type { RenderBackend, SegmentationModel, SegmentationTransform, VideoProcessor } from '@pexip/media-processor';
-import type { MediaDeviceRequest } from '@pexip/media-control';
-import type { MediaSignals, Segmenters, TrackProcessor, VideoContentHint, VideoRenderParams, VideoStreamTrackProcessorAPIs } from './types';
-interface ProcessorDeps {
-    segmenters: Partial<Segmenters>;
-    transformer?: SegmentationTransform;
-    videoProcessor?: () => VideoProcessor;
-    videoSegmentationModel?: SegmentationModel;
-}
-interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<ProcessorDeps, 'videoProcessor'> {
-    /**
-     * What API to use for processing the MediaStreamTrack
-     * `stream` - Use MediaStreamTrackProcessor, when available
-     * `canvas` - Use Canvas
-     */
-    trackProcessorAPI?: () => VideoStreamTrackProcessorAPIs;
+import type { RenderBackend, VideoProcessor } from '@pexip/media-processor';
+import type { InputConstraintSet, MediaDeviceRequest } from '@pexip/media-control';
+import type { ExtendedMediaTrackSettings, MediaSignals, TrackProcessor, VideoContentHint, VideoRenderParams } from './types';
+interface VideoStreamProcessOptions extends Partial<VideoRenderParams> {
     /**
      * Whether or to enable this processor
      */
     shouldEnable: () => boolean;
+    getVideoProcessor: () => VideoProcessor;
     referenceProcessingHeight?: number;
     processingWidth: number;
     processingHeight: number;
-    hasInitializedDeps?: boolean;
     width?: number;
     height?: number;
     label?: string;
@@ -30,13 +18,43 @@ interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<Pro
     gpuAPI?: () => RenderBackend;
     signals?: MediaSignals;
 }
-interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<ProcessorDeps> {
-    hasInitialized: boolean;
+interface VideoStreamProcessProps extends Partial<VideoRenderParams> {
     contentHint?: VideoContentHint;
 }
-declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUrl", "backgroundThreshold", "maskCombineRatio", "edgeBlurAmount", "foregroundThreshold", "frameRate", "videoSegmentation", "videoSegmentationModel", "width", "height", "pan", "tilt", "zoom", "contentHint", "sigmaSpace", "sigmaRangeLo", "sigmaRangeHi", "excludeBystanders", "personCenterX", "personCenterY", "morphErodeRadiusPx", "morphPass", "morphDilateRadiusPx", "lightWrapIntensity", "lightWrapTightness", "lightWrapEdgeBand", "lightWrapBlurAmount", "downSampleFactor"];
-type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
+declare const FEATURE_KEYS: {
+    readonly backgroundBlurAmount: "number";
+    readonly backgroundImageUrl: "string";
+    readonly backgroundThreshold: "number";
+    readonly maskCombineRatio: "number";
+    readonly edgeBlurAmount: "number";
+    readonly foregroundThreshold: "number";
+    readonly frameRate: "number";
+    readonly videoSegmentation: "string";
+    readonly width: "number";
+    readonly height: "number";
+    readonly pan: "boolean";
+    readonly tilt: "boolean";
+    readonly zoom: "boolean";
+    readonly contentHint: "string";
+    readonly sigmaSpace: "number";
+    readonly sigmaRangeLo: "number";
+    readonly sigmaRangeHi: "number";
+    readonly excludeBystanders: "boolean";
+    readonly personCenterX: "number";
+    readonly personCenterY: "number";
+    readonly morphErodeRadiusPx: "number";
+    readonly morphPass: "number";
+    readonly morphDilateRadiusPx: "number";
+    readonly lightWrapIntensity: "number";
+    readonly lightWrapTightness: "number";
+    readonly lightWrapEdgeBand: "number";
+    readonly lightWrapBlurAmount: "number";
+    readonly downSampleFactor: "number";
+};
+type FeaturePropKeys = keyof typeof FEATURE_KEYS;
 type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest["video"], props: FeatureProps) => FeatureProps;
-export declare const createVideoStreamProcess: ({ backgroundImageUrl, dynamicProcessingDimensions, backgroundThreshold, downSampleFactor, foregroundThreshold, lightWrapEdgeBand, lightWrapIntensity, lightWrapTightness, morphDilateRadiusPx, morphErodeRadiusPx, morphPass, personCenterX, personCenterY, sigmaSpace, sigmaRangeHi, sigmaRangeLo, frameRate, gpuAPI, label, referenceProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute, trackProcessorAPI, videoSegmentation, signals, ...options }: VideoStreamProcessOptions) => TrackProcessor;
+export declare const mapFeatureToSettings: (props: InputConstraintSet) => ExtendedMediaTrackSettings;
+export declare const mapSettingsToFeatures: (settings: ExtendedMediaTrackSettings) => InputConstraintSet;
+export declare const createVideoStreamProcess: ({ backgroundImageUrl, dynamicProcessingDimensions, backgroundThreshold, downSampleFactor, foregroundThreshold, lightWrapEdgeBand, lightWrapIntensity, lightWrapTightness, morphDilateRadiusPx, morphErodeRadiusPx, morphPass, personCenterX, personCenterY, sigmaSpace, sigmaRangeHi, sigmaRangeLo, frameRate, gpuAPI, label, referenceProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute, videoSegmentation, signals, getVideoProcessor, ...options }: VideoStreamProcessOptions) => TrackProcessor;
 export {};