@pexip/media 21.0.0 → 22.1.0

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 import type { IndexedDevices, InputConstraintSet, InputDeviceConstraint, MediaDeviceInfoLike, MediaDeviceRequest } from '@pexip/media-control';
-import type { RendererOptions, RenderEffects, Segmenter, SegmentationModel } from '@pexip/media-processor';
+import type { RendererOptions, RenderEffects } from '@pexip/media-processor';
 import type { Signal } from '@pexip/signal';
 export type Unsubscribe = () => void;
 export interface ExtendedMediaTrackSettings extends MediaTrackSettings {
@@ -14,10 +14,12 @@ export interface ExtendedMediaTrackSettings extends MediaTrackSettings {
     maskCombineRatio?: number;
     foregroundThreshold?: number;
     videoSegmentation?: RenderEffects;
-    videoSegmentationModel?: SegmentationModel;
-    pan?: boolean;
-    tilt?: boolean;
-    zoom?: boolean;
+    panEnabled?: boolean;
+    tiltEnabled?: boolean;
+    zoomEnabled?: boolean;
+    pan?: number;
+    tilt?: number;
+    zoom?: number;
     contentHint?: AudioContentHint | VideoContentHint;
 }
 export type ExtendedMediaTrackSettingsKey = keyof ExtendedMediaTrackSettings;
@@ -230,12 +232,6 @@ export type GetUserMediaProcess = (request: {
     };
     currentMedia: Media | undefined;
 }) => Promise<Media>;
-/**
- * Use which processor API to process the stream track
- * `stream` - Use `MediaStreamTrackProcessor`, when available
- * `canvas` - Use Canvas
- */
-export type VideoStreamTrackProcessorAPIs = 'stream' | 'canvas';
 export declare enum UserMediaStatus {
     /**
      * The initial status
@@ -573,9 +569,6 @@ export declare enum DeniedDevices {
     Camera = "camera",
     Both = "microphone-and-camera"
 }
-export type Segmenters = {
-    [Property in SegmentationModel]: Segmenter;
-};
 /**
  * Audio content hints are only applicable when the MediaStreamTrack contains an
  * audio track

package/dist/utils.js

@@ -3,6 +3,7 @@ import { calculateMaxBlurPass, clamping, } from '@pexip/media-processor';
 import { hasOwn, assert, isEmpty } from '@pexip/utils';
 import { internalSignals } from './signals';
 import { MAX_TENT_BLUR_SIZE } from './constants';
+import { mapFeatureToSettings } from './videoProcessor';
 export const makeDeriveDeviceStatus = (constraints) => (audio, video, both) => {
     if (constraints.audio) {
         if (constraints.video) {
@@ -181,9 +182,9 @@ export const createMediaTrack = (trackInit) => {
         },
         async release() {
             trackUnsubscribe?.();
-            trackInit.track?.stop();
             await trackInit.release?.();
             await trackInit.previousMediaTrack?.release();
+            trackInit.track?.stop();
             trackInit.signals?.onMediaTrackStopped?.emit(this);
         },
         toJSON() {
@@ -571,7 +572,6 @@ export const AUDIO_SETTINGS_KEYS = [
 export const VIDEO_SETTINGS_KEYS = [
     'frameRate',
     'videoSegmentation',
-    'videoSegmentationModel',
     'foregroundThreshold',
     'backgroundBlurAmount',
     'edgeBlurAmount',
@@ -629,10 +629,11 @@ export const getSettingsFromKeys = (keys, settings) => {
         return settings;
     }
     const result = {};
+    const mapped = mapFeatureToSettings(settings);
     for (const key of keys) {
-        if (settings[key] !== undefined) {
+        if (mapped[key] !== undefined) {
             // @ts-expect-error --- Type issue and need to be fixed when we have time
-            result[key] = settings[key];
+            result[key] = mapped[key];
         }
     }
     return result;
@@ -782,12 +783,21 @@ export const createMediaProcessor = ({ audioProcessors, videoProcessors, onProce
         }
         catch (error) {
             if (error instanceof Error) {
+                if (track.stopped) {
+                    // Ignore the error as track stopped
+                    return track;
+                }
                 onProcessingError?.(error, track);
             }
             return track;
         }
     };
     return async (newTracks) => {
-        return await Promise.all(newTracks.map(track => processTrack(track)));
+        return await Promise.all(newTracks.flatMap(track => {
+            if (track.stopped === undefined || track.stopped) {
+                return [];
+            }
+            return [processTrack(track)];
+        }));
     };
 };

package/dist/videoProcessor.d.ts

@@ -1,27 +1,15 @@
-import type { RenderBackend, SegmentationModel, SegmentationTransform, VideoProcessor } from '@pexip/media-processor';
-import type { MediaDeviceRequest } from '@pexip/media-control';
-import type { MediaSignals, Segmenters, TrackProcessor, VideoContentHint, VideoRenderParams, VideoStreamTrackProcessorAPIs } from './types';
-interface ProcessorDeps {
-    segmenters: Partial<Segmenters>;
-    transformer?: SegmentationTransform;
-    videoProcessor?: () => VideoProcessor;
-    videoSegmentationModel?: SegmentationModel;
-}
-interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<ProcessorDeps, 'videoProcessor'> {
-    /**
-     * What API to use for processing the MediaStreamTrack
-     * `stream` - Use MediaStreamTrackProcessor, when available
-     * `canvas` - Use Canvas
-     */
-    trackProcessorAPI?: () => VideoStreamTrackProcessorAPIs;
+import type { RenderBackend, VideoProcessor } from '@pexip/media-processor';
+import type { InputConstraintSet, MediaDeviceRequest } from '@pexip/media-control';
+import type { ExtendedMediaTrackSettings, MediaSignals, TrackProcessor, VideoContentHint, VideoRenderParams } from './types';
+interface VideoStreamProcessOptions extends Partial<VideoRenderParams> {
     /**
      * Whether or to enable this processor
      */
     shouldEnable: () => boolean;
+    getVideoProcessor: () => VideoProcessor;
     referenceProcessingHeight?: number;
     processingWidth: number;
     processingHeight: number;
-    hasInitializedDeps?: boolean;
     width?: number;
     height?: number;
     label?: string;
@@ -30,13 +18,43 @@ interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<Pro
     gpuAPI?: () => RenderBackend;
     signals?: MediaSignals;
 }
-interface VideoStreamProcessProps extends Partial<VideoRenderParams>, Required<ProcessorDeps> {
-    hasInitialized: boolean;
+interface VideoStreamProcessProps extends Partial<VideoRenderParams> {
     contentHint?: VideoContentHint;
 }
-declare const FEATURE_KEYS: readonly ["backgroundBlurAmount", "backgroundImageUrl", "backgroundThreshold", "maskCombineRatio", "edgeBlurAmount", "foregroundThreshold", "frameRate", "videoSegmentation", "videoSegmentationModel", "width", "height", "pan", "tilt", "zoom", "contentHint", "sigmaSpace", "sigmaRangeLo", "sigmaRangeHi", "excludeBystanders", "personCenterX", "personCenterY", "morphErodeRadiusPx", "morphPass", "morphDilateRadiusPx", "lightWrapIntensity", "lightWrapTightness", "lightWrapEdgeBand", "lightWrapBlurAmount", "downSampleFactor"];
-type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
+declare const FEATURE_KEYS: {
+    readonly backgroundBlurAmount: "number";
+    readonly backgroundImageUrl: "string";
+    readonly backgroundThreshold: "number";
+    readonly maskCombineRatio: "number";
+    readonly edgeBlurAmount: "number";
+    readonly foregroundThreshold: "number";
+    readonly frameRate: "number";
+    readonly videoSegmentation: "string";
+    readonly width: "number";
+    readonly height: "number";
+    readonly pan: "boolean";
+    readonly tilt: "boolean";
+    readonly zoom: "boolean";
+    readonly contentHint: "string";
+    readonly sigmaSpace: "number";
+    readonly sigmaRangeLo: "number";
+    readonly sigmaRangeHi: "number";
+    readonly excludeBystanders: "boolean";
+    readonly personCenterX: "number";
+    readonly personCenterY: "number";
+    readonly morphErodeRadiusPx: "number";
+    readonly morphPass: "number";
+    readonly morphDilateRadiusPx: "number";
+    readonly lightWrapIntensity: "number";
+    readonly lightWrapTightness: "number";
+    readonly lightWrapEdgeBand: "number";
+    readonly lightWrapBlurAmount: "number";
+    readonly downSampleFactor: "number";
+};
+type FeaturePropKeys = keyof typeof FEATURE_KEYS;
 type FeatureProps = Pick<Partial<VideoStreamProcessProps>, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest["video"], props: FeatureProps) => FeatureProps;
-export declare const createVideoStreamProcess: ({ backgroundImageUrl, dynamicProcessingDimensions, backgroundThreshold, downSampleFactor, foregroundThreshold, lightWrapEdgeBand, lightWrapIntensity, lightWrapTightness, morphDilateRadiusPx, morphErodeRadiusPx, morphPass, personCenterX, personCenterY, sigmaSpace, sigmaRangeHi, sigmaRangeLo, frameRate, gpuAPI, label, referenceProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute, trackProcessorAPI, videoSegmentation, signals, ...options }: VideoStreamProcessOptions) => TrackProcessor;
+export declare const mapFeatureToSettings: (props: InputConstraintSet) => ExtendedMediaTrackSettings;
+export declare const mapSettingsToFeatures: (settings: ExtendedMediaTrackSettings) => InputConstraintSet;
+export declare const createVideoStreamProcess: ({ backgroundImageUrl, dynamicProcessingDimensions, backgroundThreshold, downSampleFactor, foregroundThreshold, lightWrapEdgeBand, lightWrapIntensity, lightWrapTightness, morphDilateRadiusPx, morphErodeRadiusPx, morphPass, personCenterX, personCenterY, sigmaSpace, sigmaRangeHi, sigmaRangeLo, frameRate, gpuAPI, label, referenceProcessingHeight, maskCombineRatio, processingHeight, processingWidth, shouldEnable, stopAsMute, videoSegmentation, signals, getVideoProcessor, ...options }: VideoStreamProcessOptions) => TrackProcessor;
 export {};