@pexip/media 18.3.0 → 18.5.0

package/CHANGELOG.md

@@ -1,5 +1,43 @@
 # @pexip/media
 
+## 18.5.0
+
+### Minor Changes
+
+- f146682: Make `delegate` prop to be a fn
+
+### Patch Changes
+
+- d98fe9d: Move to module type.
+- Updated dependencies [d98fe9d]
+- Updated dependencies [acd7aba]
+- Updated dependencies [f146682]
+  - @pexip/media-processor@18.5.0
+  - @pexip/media-control@18.5.0
+  - @pexip/signal@16.7.1
+  - @pexip/utils@16.12.1
+
+## 18.4.0
+
+### Patch Changes
+
+- 9179b97: Added hasPtzFeature function to utils.ts
+
+  Added isDevicesInUse function to status.ts
+
+  Added isTryingToRequestSameUsedAudioDevice /
+  isTryingToRequestSameUsedVideoDevice check to updateMedia function in
+  media.ts. This avoids infinity reloads in Firefox if a blocked (e.g. by
+  another application) device is selected.
+
+- Updated dependencies [36d5672]
+- Updated dependencies [4bfce08]
+- Updated dependencies [31e4c17]
+  - @pexip/media-processor@18.4.0
+  - @pexip/media-control@18.4.0
+  - @pexip/signal@16.7.0
+  - @pexip/utils@16.12.0
+
 ## 18.3.0
 
 ### Minor Changes

package/README.md

@@ -53,21 +53,6 @@ that's specified from the
 [Media Capture and Streams](https://w3c.github.io/mediacapture-main) spec, we
 have extended some additional media features as the following:
 
-- Able to pass `MediaDeviceInfo` directly to audio/video
-- Add `vad` to use our own implementation of Voice Activity Detection feature
-- Add `asd` to use our own implementation of Audio Signal Detection feature
-- Add `denoise` to use our own implementation of noise suppression
-- Add `backgroundBlurAmount` to adjust the blur amount when using the background
-  blur feature
-- Add `edgeBlurAmount` to adjust the blur amount for edge of the segmented
-  person
-- Add `foregroundThreshold` to adjust the threshold to be considered as
-  foreground aka the segmented person
-- Add `bgImageUrl` for the image to be used for background replacement
-- Add `videoSegmentation` for specified effects of video segmentation
-- Add `videoSegmentationModel` for specifying segmentation model to be used
-- Add `flipHorizontal` to flip horizontally
-
 ```typescript
 export interface InputConstraintSet extends MediaTrackConstraints {
   /**
@@ -100,9 +85,10 @@ export interface InputConstraintSet extends MediaTrackConstraints {
    */
   asd?: ConstrainBoolean;
   /**
-   * Flip the video horizontally
+   * Audio Signal Detection for the purpose of checking if the audio input is
+   * hardware muted or unusable
    */
-  flipHorizontal?: ConstrainBoolean;
+  mixWithAdditionalMedia?: ConstrainBoolean;
   /**
    * Blur size/level parameter when using video segmentation with `blur`
    * effects
@@ -119,7 +105,26 @@ export interface InputConstraintSet extends MediaTrackConstraints {
   /**
    * Image Url that is being used for video overlay effects
    */
-  bgImageUrl?: ConstrainDOMString;
+  backgroundImageUrl?: ConstrainDOMString;
+  /**
+   * The ratio to be used for smoothing segmentation mask
+   */
+  maskCombineRatio?: ConstrainDouble;
+  /**
+   * Image Url that is being used for video overlay effects
+   */
+  resizeMode?: ConstrainDOMString;
+  /**
+   * Current pan, tilt and zoom level for a PTZ controllable camera on web
+   */
+  pan?: boolean;
+  tilt?: boolean;
+  zoom?: boolean;
+
+  /**
+   * Content Hint for the track to apply
+   */
+  contentHint?: ConstrainDOMString;
 }
 ```
 
@@ -127,60 +132,35 @@ export interface InputConstraintSet extends MediaTrackConstraints {
 
 ```typescript
 import {createMedia, createMediaSignals, UserMediaStatus} from '@pexip/media';
-import type {VideoRenderParams} from '@pexip/media';
+import type {RenderEffects, Segmenters, Media} from '@pexip/media';
 import {
-  urls as mpUrls,
-  createMediapipeSegmenter,
-  createPsySegSegmenter,
+  createSegmenter,
   createCanvasTransform,
-  createVideoTrackProcessor,
-  createVideoTrackProcessorWithFallback,
   createVideoProcessor,
-  PROCESSING_WIDTH,
-  PROCESSING_HEIGHT,
-  FOREGROUND_THRESHOLD,
-  BACKGROUND_BLUR_AMOUNT,
-  EDGE_BLUR_AMOUNT,
-  FLIP_HORIZONTAL,
 } from '@pexip/media-processor';
 
-// Need some state for the app
-interface State extends VideoRenderParams {
-  mute: {
-    audio: boolean;
-    video: boolean;
-  };
-  noiseSuppression: boolean;
-  vad: boolean;
-  asd: boolean;
-  updateFrequency: number;
-  silentDetectionDuration: number;
-  vadThrottleMS: number;
-  monitor: boolean;
-  channelCount: number;
-}
-export const mediaState: State = {
-  mute: {
+// App states
+const states: {
+  muted: {video: boolean; audio: boolean};
+  effects: RenderEffects;
+  foregroundThreshold: number;
+  backgroundBlurAmount: number;
+  maskCombineRatio: number;
+  backgroundImageUrl: string;
+  media?: Media;
+  status: UserMediaStatus;
+} = {
+  muted: {
     audio: false,
     video: false,
   },
-  videoSegmentation: 'none',
-  noiseSuppression: false,
-  vad: false,
-  asd: false,
-  updateFrequency: 0.5,
-  silentDetectionDuration: 4.0,
-  vadThrottleMS: 3000,
-  width: PROCESSING_WIDTH,
-  height: PROCESSING_HEIGHT,
-  frameRate: 30,
-  bgImageUrl: '',
-  monitor: false,
-  channelCount: 1,
-  foregroundThreshold: FOREGROUND_THRESHOLD,
-  backgroundBlurAmount: BACKGROUND_BLUR_AMOUNT,
-  edgeBlurAmount: EDGE_BLUR_AMOUNT,
-  flipHorizontal: FLIP_HORIZONTAL,
+  // Could be `none`, `blur` or `overlay`
+  effects: 'none',
+  foregroundThreshold: 0.5, // 0.0 - 1.0
+  backgroundBlurAmount: 30, // 0 - 100
+  maskCombineRatio: 0.5, // 0.0 - 1.0
+  backgroundImageUrl: '/some/path/to/an/image/for/background/replacement.png',
+  status: UserMediaStatus.Initial,
 };
 
 // Create required signals with additional `onDevicesChanged`,
@@ -191,71 +171,61 @@ export const signals = createMediaSignals('demo', [
   'onStreamTrackEnabled',
 ]);
 
-const audioProcessor = createAudioStreamProcess({
-  shouldEnable: () => true,
-  denoiseParams: {
-    wasmURL: denoiseWasm.href,
-    workletModule: mpUrls.denoise.href,
-  },
-  analyzerUpdateFrequency: mediaState.updateFrequency,
-  audioSignalDetectionDuration: mediaState.silentDetectionDuration,
-  throttleMs: mediaState.vadThrottleMS,
-  onAudioSignalDetected: signals.onSilentDetected.emit,
-  onVoiceActivityDetected: signals.onVAD.emit,
+// Setting the path to that `@mediapipe/tasks-vision` assets
+// It will be passed direct to
+// [FilesetResolver.forVisionTasks()](https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.filesetresolver#filesetresolverforvisiontasks)
+const tasksVisionBasePath =
+  'A base path to specify the directory the Wasm files should be loaded from';
+
+const modelAsset = {
+  /**
+   * Path to mediapipe selfie segmentation model asset
+   */
+  path: 'A path to selfie segmentation model',
+  modelName: 'selfie' as const,
+};
+
+const segmenter = createSegmenter(tasksVisionBasePath, {modelAssets});
+// Create a processing transformer and set the effects to `blur`
+const transformer = createCanvasTransform(segmenter, {
+  effects: states.effects,
+  foregroundThreshold: states.foregroundThreshold,
+  backgroundBlurAmount: states.backgroundBlurAmount,
+  maskCombineRatio: states.maskCombineRatio,
+  backgroundImageUrl: states.backgroundImageUrl,
 });
 
-// Create symbolic link to link bg-blur/selfie_segmentation to a public folder
-// Thus we can create a URL object to get the link to be used for the `gluePath`
-const selfieJs = new URL(
-  `./selfie_segmentation/selfie_segmentation.js`,
-  document.baseURI,
-);
-const mediapipeSegmenter = createMediapipeSegmenter('selfie_segmentation', {
-  modelType: 'general',
-  processingWidth: PROCESSING_WIDTH,
-  processingHeight: PROCESSING_HEIGHT,
-  gluePath: selfieJs.href,
+const delegate = () => (isChromium() ? 'GPU' : 'CPU');
+const selfie = createSegmenter(taskVisionBasePath, {
+  modelAsset,
+  delegate,
 });
 
-const segmenters: Segmenters = {
-  mediapipeSelfie: mediapipeSegmenter,
+export const segmenters: Partial<Segmenters> = {
+  selfie,
 };
 
-const transformer = createCanvasTransform(mediapipeSegmenter, {
-  effects: mediaState.videoSegmentation,
-  width: mediaState.width,
-  height: mediaState.height,
-  foregroundThreshold: mediaState.foregroundThreshold,
-  backgroundBlurAmount: mediaState.backgroundBlurAmount,
-  edgeBlurAmount: mediaState.edgeBlurAmount,
-  flipHorizontal: mediaState.flipHorizontal,
-});
 const videoProcessor = createVideoProcessor([transformer]);
 
 const videoStreamProcessor = createVideoStreamProcess({
-  trackProcessorAPI: 'stream', // Use Stream API when available
   shouldEnable: () => true, // Always Process Video Track
-  videoSegmentationModel: 'mediapipeSelfie', // Segmentation Modal
   segmenters,
   transformer,
   videoProcessor,
-  processingWidth: mediaState.width,
-  processingHeight: mediaState.height,
-  onError: error => {
-    console.error(error);
-  },
 });
 
 // Instantiate the media object
 export const mediaController = createMedia({
-  getMuteState: () => mediaState.mute,
+  getMuteState: () => states.muted,
   signals,
-  mediaProcessors: [videoStreamProcessor, audioProcessor],
+  mediaProcessors: [videoStreamProcessor],
 });
 
 // Hook-up the signals to get the update
 // Subscribe the onMediaChanged signal to get the latest Media object change event
-signals.onMediaChanged.add(media => setLocalMedia(media)); // Assume there is a local function `setLocalMedia`
+signals.onMediaChanged.add(media => {
+  states.media = media;
+}); // Assume there is a local function `setLocalMedia`
 // Subscribe the onStatusChanged signal to get the latest Media status change event
 signals.onStatusChanged.add(media => setMediaStatus(media)); // Assume there is a local function `setMediaStatus`
 // Subscribe onStreamTrackEnabled signal to get the MediaStreamTrack['enabled'] change event
@@ -265,16 +235,13 @@ signals.onStreamTrackEnabled.add(track => {
 
 // Later we can initialize a gUM to get a MediaStream
 mediaController.getUserMedia({
-  audio: {
-    sampleRate: 48000, // Ideal sample rate for our own noise suppression implementation
-    denoise: true, // Use our own implementation of noise suppression
-    noiseSuppression: false, // Disable the built-in noise suppression
-    vad: true, // Use Voice Activity Detection
-    asd: true, // Use Audio Signal Detection to check if the microphone is malfunctioned
-  },
+  audio: true,
   video: {
     videoSegmentation: 'blur', // Use background blur
-    blurLevel: 21,
+    foregroundThreshold: states.foregroundThreshold,
+    backgroundBlurAmount: states.backgroundBlurAmount,
+    maskCombineRatio: states.maskCombineRatio,
+    backgroundImageUrl: states.backgroundImageUrl,
   },
 });
 
@@ -314,39 +281,3 @@ await mediaController.media.applyConstraints({
   audio: {denoise: false, noiseSuppression: true},
 });
 ```
-
-### `createPreviewController`
-
-```typescript
-const {createPreviewController} = require('@pexip/media');
-
-// Some states from the App
-interface AppState {
-  media: Media;
-  devices: MediaDeviceInfo[];
-}
-// Some functions from the App
-const updateStream = (constraints: MediaDeviceRequest) => Promise<void>;
-
-const onMediaChangedSignal = createSignal<Media>();
-
-// Instantiate the controller
-const controller = createPreviewController({
-  getDesiredResolution: () => ({}),
-  getCurrentDevices: () => state.devices,
-  getCurrentMedia: () => state.media,
-  updateMainStream: updateStream,
-  mediaSignal: onMediaChangedSignal,
-  getSegmentationOptions: () => ({
-    effects: mediaState.videoSegmentation,
-    erode: mediaState.erode,
-    blurLevel: mediaState.blurLevel,
-  }),
-  videoProcessingEnabled: true,
-  updateMainStream: mc.getUserMediaAsync,
-  onEnded: () => {
-    close();
-  },
-  psySegScriptLoaded: true,
-});
-```

package/dist/media.js

@@ -1,10 +1,10 @@
-import { subscribe, MediaEventType, MediaDeviceFailure, createStreamTrackEventSubscriptions, getDevices, isRequestedResolution, mergeConstraints, shouldRequestDevice, } from '@pexip/media-control';
+import { subscribe, MediaEventType, MediaDeviceFailure, createStreamTrackEventSubscriptions, getDevices, isRequestedResolution, mergeConstraints, shouldRequestDevice, extractConstrainDevice, } from '@pexip/media-control';
 import { createAsyncQueue, isEmpty } from '@pexip/utils';
-import { getPermissionStatus, deriveInitialPermissionStatus, isInitialPermissions, isInitialPermissionsGranted, } from './status';
+import { getPermissionStatus, deriveInitialPermissionStatus, isInitialPermissions, isInitialPermissionsGranted, isVideoDeviceInUse, isDevicesInUse, isAudioDeviceInUse, } from './status';
 import { UserMediaStatus } from './types';
 import { createModuleLogger, logger } from './logger';
 import { createGetUserMediaProcess, requestUserMediaWithRetry, } from './userMedia';
-import { createMediaPipeline, createMediaProcess, buildMedia, getDevicesChanges, shallowCopy, wrapToJSON, hasSettingsChanged, AUDIO_SETTINGS_KEYS, VIDEO_SETTINGS_KEYS, MIXING_SETTINGS_KEYS, applyContentHint, } from './utils';
+import { createMediaPipeline, createMediaProcess, buildMedia, getDevicesChanges, shallowCopy, wrapToJSON, hasSettingsChanged, AUDIO_SETTINGS_KEYS, VIDEO_SETTINGS_KEYS, MIXING_SETTINGS_KEYS, applyContentHint, hasPtzFeature, } from './utils';
 import { isMedia } from './typeGuard';
 import { updateFeatureProps as getVideoFeatures } from './videoProcessor';
 import { updateFeatureProps as getAudioFeatures } from './audioProcessor';
@@ -171,19 +171,35 @@ export const createMedia = ({ getMuteState, signals, mediaProcessors, getDefault
         if (!constraints.audio && !constraints.video) {
             throw new Error(MediaDeviceFailure.MissingConstraintsError);
         }
-        const shouldRequestAudio = shouldRequestDevice(constraints.audio, props.media?.rawStream?.getAudioTracks() ?? [], props.devices.filter(device => device.kind === 'audioinput'));
-        const shouldRequestVideo = shouldRequestDevice(constraints.video, props.media?.rawStream?.getVideoTracks() ?? [], props.devices.filter(device => device.kind === 'videoinput'));
+        // Check if the audio device was requested before and is currently in use by another application
+        const isTryingToRequestSameUsedAudioDevice = (isAudioDeviceInUse(props.media.status) ||
+            isDevicesInUse(props.media.status)) &&
+            props.media.expectedAudioInput?.deviceId &&
+            props.media.expectedAudioInput?.deviceId ===
+                extractConstrainDevice(constraints.audio)?.[0]?.[0]?.deviceId;
+        // Check if the video device was requested before and is currently in use by another application
+        const isTryingToRequestSameUsedVideoDevice = (isVideoDeviceInUse(props.media.status) ||
+            isDevicesInUse(props.media.status)) &&
+            props.media.expectedVideoInput?.deviceId &&
+            props.media.expectedVideoInput?.deviceId ===
+                extractConstrainDevice(constraints?.video)?.[0]?.[0]?.deviceId;
+        const shouldRequestAudio = !isTryingToRequestSameUsedAudioDevice &&
+            shouldRequestDevice(constraints.audio, props.media?.rawStream?.getAudioTracks() ?? [], props.devices.filter(device => device.kind === 'audioinput'));
+        const shouldRequestVideo = !isTryingToRequestSameUsedVideoDevice &&
+            shouldRequestDevice(constraints.video, props.media?.rawStream?.getVideoTracks() ?? [], props.devices.filter(device => device.kind === 'videoinput'));
         const { audio: [prevAudioSettings], video: [prevVideoSettings], } = props.media.getSettings();
         const videoFeatures = getVideoFeatures(constraints.video, {});
         const audioFeatures = getAudioFeatures(constraints.audio, {});
         const mixingFeatures = getMixingFeatures(constraints.audio, {});
         const hasRequestedResolution = ['blur', 'overlay'].includes(videoFeatures.videoSegmentation ?? '') || // Skip when there is a render effect to modify the video
             !props.devices.some(device => device.kind === 'videoinput' && device.label) || // Skip when no authorized video found
+            isTryingToRequestSameUsedVideoDevice ||
             isRequestedResolution(constraints.video, props.media?.videoInput);
         const audioFeaturesChanged = hasSettingsChanged(AUDIO_SETTINGS_KEYS)(prevAudioSettings, audioFeatures);
         const videoFeaturesChanged = hasSettingsChanged(VIDEO_SETTINGS_KEYS)(prevVideoSettings, videoFeatures);
         const mixingFeaturesChanged = hasSettingsChanged(MIXING_SETTINGS_KEYS)(prevAudioSettings, mixingFeatures);
-        const ptzFeaturesChanges = hasSettingsChanged(['pan', 'tilt', 'zoom'])(prevVideoSettings, videoFeatures);
+        const ptzFeaturesChanges = hasPtzFeature() &&
+            hasSettingsChanged(['pan', 'tilt', 'zoom'])(prevVideoSettings, videoFeatures);
         logger.debug({
             constraints,
             stream: props.media.stream,

package/dist/status.d.ts

@@ -29,6 +29,7 @@ export declare const isInitialPermissionsNotGranted: UserMediaValidator;
 export declare const isInitialPermissionsGranted: UserMediaValidator;
 export declare const isAudioDeviceInUse: UserMediaValidator;
 export declare const isDeviceInUse: UserMediaValidator;
+export declare const isDevicesInUse: UserMediaValidator;
 export declare const isVideoDeviceInUse: UserMediaValidator;
 export declare const toDeniedDevices: (status?: UserMediaStatus) => DeniedDevices | undefined;
 export declare const getPermissionStatus: (getPermissionState?: (anyActiveStream?: boolean | undefined) => Promise<import("@pexip/media-control").InputDevicePermission>) => Promise<UserMediaStatus.Initial | UserMediaStatus.InitialPermissionsGranted | UserMediaStatus.InitialPermissionsNotGranted | UserMediaStatus.InitialPermissionsVideoInputDenied | UserMediaStatus.InitialPermissionsAudioInputDenied | UserMediaStatus.InitialPermissionsVideoInputGranted | UserMediaStatus.InitialPermissionsAudioInputGranted | UserMediaStatus.InitialPermissionsGrantedVideoInputDenied | UserMediaStatus.InitialPermissionsGrantedAudioInputDenied | UserMediaStatus.PermissionsRejected>;

package/dist/status.js

@@ -121,6 +121,7 @@ export const isDeviceInUse = status => [
     UserMediaStatus.AudioDeviceInUse,
     UserMediaStatus.VideoDeviceInUse,
 ].includes(status);
+export const isDevicesInUse = status => status === UserMediaStatus.DevicesInUse;
 export const isVideoDeviceInUse = status => status === UserMediaStatus.VideoDeviceInUse;
 export const toDeniedDevices = (status) => {
     if (!status) {

package/dist/utils.d.ts

@@ -1,4 +1,4 @@
-import type { MediaDeviceRequest, MediaDeviceInfoLike } from '@pexip/media-control';
+import type { MediaDeviceInfoLike, MediaDeviceRequest } from '@pexip/media-control';
 import { UserMediaStatus } from './types';
 import type { Media, MediaAttributes, Pipeline, Process, ProcessMedia, ExtendedMediaTrackSettingsKey, ExtendedMediaTrackSettings } from './types';
 export declare const makeDeriveDeviceStatus: (constraints: MediaDeviceRequest) => (audio: UserMediaStatus, video: UserMediaStatus, both: UserMediaStatus) => UserMediaStatus;
@@ -105,4 +105,9 @@ export declare const getBlurKernelSize: (percentage: number, height: number, max
  * @param track - The track to be applied
  */
 export declare const applyContentHint: <T extends "" | "speech" | "speech-recognition" | "music" | "motion" | "detail" | "text">(hint?: T | undefined) => (track: MediaStreamTrack) => void;
+/**
+ * Check if the browser supports PTZ feature
+ * @returns true if the browser supports PTZ feature, otherwise false
+ */
+export declare const hasPtzFeature: () => boolean;
 export {};

package/dist/utils.js

@@ -1,5 +1,6 @@
 import { applyConstraints, createTrackDevicesChanges, extractConstraintsWithKeys, findDeviceFromConstraints, muteStreamTrack, relaxInputConstraint, stopMediaStream, isAudioInput, isVideoInput, } from '@pexip/media-control';
 import { calculateMaxBlurPass } from '@pexip/media-processor';
+import { hasOwn } from '@pexip/utils';
 import { UserMediaStatus } from './types';
 import { isOverConstrained } from './status';
 import { isMedia } from './typeGuard';
@@ -359,3 +360,13 @@ export const applyContentHint = (hint) => (track) => {
         track.contentHint = hint;
     }
 };
+/**
+ * Check if the browser supports PTZ feature
+ * @returns true if the browser supports PTZ feature, otherwise false
+ */
+export const hasPtzFeature = () => {
+    const supports = navigator.mediaDevices.getSupportedConstraints();
+    return (hasOwn(supports, 'pan') &&
+        hasOwn(supports, 'tilt') &&
+        hasOwn(supports, 'zoom'));
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "18.3.0",
+  "version": "18.5.0",
   "description": "Home for media related stuff",
   "homepage": "https://gitlab.com/pexip/zoo",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -9,6 +9,7 @@
     "url": "https://gitlab.com/pexip/zoo.git",
     "directory": "src/aquila/packages/media"
   },
+  "type": "module",
   "license": "Apache-2.0",
   "author": "Terrence Lam <terrence@tlam.dev>",
   "module": "dist/index.js",
@@ -25,21 +26,24 @@
     "clean": "rm -fr dist",
     "dev": "pexip-bundler dev",
     "prepack": "yarn clean && yarn build --no-sourcemap",
-    "test": "pexip-bundler test",
+    "test": "jest",
     "check-format": "prettier --ignore-path=../../.prettierignore --check .",
     "format": "prettier --ignore-path=../../.prettierignore --write .",
     "lint": "yarn run -T eslint --ext .js,.ts,.tsx --report-unused-disable-directives .",
     "typecheck": "yarn tsc --noEmit -p ."
   },
   "dependencies": {
-    "@pexip/media-control": "18.3.0",
-    "@pexip/media-processor": "18.3.0",
-    "@pexip/signal": "16.7.0",
-    "@pexip/utils": "16.12.0"
+    "@pexip/media-control": "18.5.0",
+    "@pexip/media-processor": "18.5.0",
+    "@pexip/signal": "16.7.1",
+    "@pexip/utils": "16.12.1"
   },
   "devDependencies": {
-    "@jest/globals": "^29.7.0",
-    "@pexip/bundler": "16.6.1",
+    "@pexip/bundler": "17.0.0",
+    "@swc/core": "^1.4.6",
+    "@swc/jest": "^0.2.36",
+    "jest": "^29.5.12",
+    "jest-junit": "^16.0.0",
     "prettier": "^3.2.5",
     "typescript": "~5.3.0"
   },