@pexip/media 16.7.1

package/LICENSE

@@ -0,0 +1,188 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+    "License" shall mean the terms and conditions for use, reproduction, and
+    distribution as defined by Sections 1 through 9 of this document.
+
+    "Licensor" shall mean the copyright owner or entity authorized by the
+    copyright owner that is granting the License.
+
+    "Legal Entity" shall mean the union of the acting entity and all other
+    entities that control, are controlled by, or are under common control with
+    that entity. For the purposes of this definition, "control" means (i) the
+    power, direct or indirect, to cause the direction or management of such
+    entity, whether by contract or otherwise, or (ii) ownership of fifty percent
+    (50%) or more of the outstanding shares, or (iii) beneficial ownership of
+    such entity.
+
+    "You" (or "Your") shall mean an individual or Legal Entity exercising
+    permissions granted by this License.
+
+    "Source" form shall mean the preferred form for making modifications,
+    including but not limited to software source code, documentation source, and
+    configuration files.
+
+    "Object" form shall mean any form resulting from mechanical transformation
+    or translation of a Source form, including but not limited to compiled
+    object code, generated documentation, and conversions to other media types.
+
+    "Work" shall mean the work of authorship, whether in Source or Object form,
+    made available under the License, as indicated by a copyright notice that is
+    included in or attached to the work (an example is provided in the Appendix
+    below).
+
+    "Derivative Works" shall mean any work, whether in Source or Object form,
+    that is based on (or derived from) the Work and for which the editorial
+    revisions, annotations, elaborations, or other modifications represent, as a
+    whole, an original work of authorship. For the purposes of this License,
+    Derivative Works shall not include works that remain separable from, or
+    merely link (or bind by name) to the interfaces of, the Work and Derivative
+    Works thereof.
+
+    "Contribution" shall mean any work of authorship, including the original
+    version of the Work and any modifications or additions to that Work or
+    Derivative Works thereof, that is intentionally submitted to Licensor for
+    inclusion in the Work by the copyright owner or by an individual or Legal
+    Entity authorized to submit on behalf of the copyright owner. For the
+    purposes of this definition, "submitted" means any form of electronic,
+    verbal, or written communication sent to the Licensor or its
+    representatives, including but not limited to communication on electronic
+    mailing lists, source code control systems, and issue tracking systems that
+    are managed by, or on behalf of, the Licensor for the purpose of discussing
+    and improving the Work, but excluding communication that is conspicuously
+    marked or otherwise designated in writing by the copyright owner as "Not a
+    Contribution."
+
+    "Contributor" shall mean Licensor and any individual or Legal Entity on
+    behalf of whom a Contribution has been received by Licensor and subsequently
+    incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this
+   License, each Contributor hereby grants to You a perpetual, worldwide,
+   non-exclusive, no-charge, royalty-free, irrevocable copyright license to
+   reproduce, prepare Derivative Works of, publicly display, publicly perform,
+   sublicense, and distribute the Work and such Derivative Works in Source or
+   Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License,
+   each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
+   no-charge, royalty-free, irrevocable (except as stated in this section)
+   patent license to make, have made, use, offer to sell, sell, import, and
+   otherwise transfer the Work, where such license applies only to those patent
+   claims licensable by such Contributor that are necessarily infringed by their
+   Contribution(s) alone or by combination of their Contribution(s) with the
+   Work to which such Contribution(s) was submitted. If You institute patent
+   litigation against any entity (including a cross-claim or counterclaim in a
+   lawsuit) alleging that the Work or a Contribution incorporated within the
+   Work constitutes direct or contributory patent infringement, then any patent
+   licenses granted to You under this License for that Work shall terminate as
+   of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or
+   Derivative Works thereof in any medium, with or without modifications, and in
+   Source or Object form, provided that You meet the following conditions:
+
+    (a) You must give any other recipients of the Work or Derivative Works a
+    copy of this License; and
+
+    (b) You must cause any modified files to carry prominent notices stating
+    that You changed the files; and
+
+    (c) You must retain, in the Source form of any Derivative Works that You
+    distribute, all copyright, patent, trademark, and attribution notices from
+    the Source form of the Work, excluding those notices that do not pertain to
+    any part of the Derivative Works; and
+
+    (d) If the Work includes a "NOTICE" text file as part of its distribution,
+    then any Derivative Works that You distribute must include a readable copy
+    of the attribution notices contained within such NOTICE file, excluding
+    those notices that do not pertain to any part of the Derivative Works, in at
+    least one of the following places: within a NOTICE text file distributed as
+    part of the Derivative Works; within the Source form or documentation, if
+    provided along with the Derivative Works; or, within a display generated by
+    the Derivative Works, if and wherever such third-party notices normally
+    appear. The contents of the NOTICE file are for informational purposes only
+    and do not modify the License. You may add Your own attribution notices
+    within Derivative Works that You distribute, alongside or as an addendum to
+    the NOTICE text from the Work, provided that such additional attribution
+    notices cannot be construed as modifying the License.
+
+    You may add Your own copyright statement to Your modifications and may
+    provide additional or different license terms and conditions for use,
+    reproduction, or distribution of Your modifications, or for any such
+    Derivative Works as a whole, provided Your use, reproduction, and
+    distribution of the Work otherwise complies with the conditions stated in
+    this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any
+   Contribution intentionally submitted for inclusion in the Work by You to the
+   Licensor shall be under the terms and conditions of this License, without any
+   additional terms or conditions. Notwithstanding the above, nothing herein
+   shall supersede or modify the terms of any separate license agreement you may
+   have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names,
+   trademarks, service marks, or product names of the Licensor, except as
+   required for reasonable and customary use in describing the origin of the
+   Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in
+   writing, Licensor provides the Work (and each Contributor provides its
+   Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied, including, without limitation, any
+   warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or
+   FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining
+   the appropriateness of using or redistributing the Work and assume any risks
+   associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in
+   tort (including negligence), contract, or otherwise, unless required by
+   applicable law (such as deliberate and grossly negligent acts) or agreed to
+   in writing, shall any Contributor be liable to You for damages, including any
+   direct, indirect, special, incidental, or consequential damages of any
+   character arising as a result of this License or out of the use or inability
+   to use the Work (including but not limited to damages for loss of goodwill,
+   work stoppage, computer failure or malfunction, or any and all other
+   commercial damages or losses), even if such Contributor has been advised of
+   the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or
+   Derivative Works thereof, You may choose to offer, and charge a fee for,
+   acceptance of support, warranty, indemnity, or other liability obligations
+   and/or rights consistent with this License. However, in accepting such
+   obligations, You may act only on Your own behalf and on Your sole
+   responsibility, not on behalf of any other Contributor, and only if You agree
+   to indemnify, defend, and hold each Contributor harmless for any liability
+   incurred by, or claims asserted against, such Contributor by reason of your
+   accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+Copyright 2023 Pexip AS
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.

package/README.md

@@ -0,0 +1,350 @@
+# `@pexip/media`
+
+A package to connect `@pexip/media-control` and `@pexip/media-processor` to
+create a streamlined media process
+
+## Overview
+
+### `createMedia`
+
+It create an object to interact with the media stream, which is usually used for
+our main stream.
+
+Its majority features:
+
+- Create a media pipeline to get and process the `MediaStream`, see
+  `createMediaPipeline`. It also provides some media features including:
+
+      - Video Processing - background blur
+      - Audio Processing - Noise Suppression
+      - Audio Signal Detection - To detect if the active audio input device capturing any audio
+      - Voice Activity Detection - To detect if there is any voice activity
+
+- Provide media input related information
+- Verify if we need to request a new `MediaStream`, see `updateMedia`
+- Subscribe events from `@pexip/media-control` and update the state accordingly
+
+### `createAudioStreamProcess`
+
+A processor to process audio from the stream to be used together with
+`createMedia` as one of the media processor of the pipeline.
+
+### `createVideoStreamProcess`
+
+A processor to process video from the stream to be used together with
+`createMedia` as one of the media processor of the pipeline.
+
+### `createPreviewController`
+
+It create an object to control the provided media stream, e.g. change the
+audio/video input device and apply changes to main stream when necessary.
+
+## Usage
+
+The package tries to follow the
+[MediaStream API's Constraints pattern](https://developer.mozilla.org/en-US/docs/Web/API/Media_Streams_API/Constraints).
+Beside the
+[MediaTrackConstraintSet](https://w3c.github.io/mediacapture-main/#dictionary-mediatrackconstraints-members)
+that's specified from the
+[Media Capture and Streams](https://w3c.github.io/mediacapture-main) spec, we
+have extended some additional media features as the following:
+
+- Able to pass `MediaDeviceInfo` directly to audio/video
+- Add `vad` to use our own implementation of Voice Activity Detection feature
+- Add `asd` to use our own implementation of Audio Signal Detection feature
+- Add `denoise` to use our own implementation of noise suppression
+- Add `backgroundBlurAmount` to adjust the blur amount when using the background
+  blur feature
+- Add `edgeBlurAmount` to adjust the blur amount for edge of the segmented
+  person
+- Add `foregroundThreshold` to adjust the threshold to be considered as
+  foreground aka the segmented person
+- Add `bgImageUrl` for the image to be used for background replacement
+- Add `videoSegmentation` for specified effects of video segmentation
+- Add `videoSegmentationModel` for specifying segmentation model to be used
+- Add `flipHorizontal` to flip horizontally
+
+```typescript
+export interface InputConstraintSet extends MediaTrackConstraints {
+  /**
+   * Same purpose as `deviceId` but it gives more information about the device
+   * so that we can have extra tolerance on device selection
+   */
+  device?: DeviceConstraint | ConstraintDeviceParameters;
+  /**
+   * Whether or not using video segmentation, e.g. background
+   * blur/replacement, to specify the effects, intended to be applied to the
+   * segment. Available effects are `none`, `blur`, `overlay` or `remove`
+   */
+  videoSegmentation?: ConstrainDOMString;
+  /**
+   * Segmentation model to be used for video segmentation, currently only
+   * supports `mediapipeSelfie` and `personify`
+   */
+  videoSegmentationModel?: ConstrainDOMString;
+  /**
+   * Whether or not using our own noise suppression
+   */
+  denoise?: ConstrainBoolean;
+  /**
+   * Voice Activity Detection
+   */
+  vad?: ConstrainBoolean;
+  /**
+   * Audio Signal Detection for the purpose of checking if the audio input is
+   * hardware muted or unusable
+   */
+  asd?: ConstrainBoolean;
+  /**
+   * Flip the video horizontally
+   */
+  flipHorizontal?: ConstrainBoolean;
+  /**
+   * Blur size/level parameter when using video segmentation with `blur`
+   * effects
+   */
+  backgroundBlurAmount?: ConstrainULong;
+  /**
+   * Blur amount applied to the segmented person's edge
+   */
+  edgeBlurAmount?: ConstrainULong;
+  /**
+   * Erode level for edge smoothing when using video segmentation
+   */
+  foregroundThreshold?: ConstrainDouble;
+  /**
+   * Image Url that is being used for video overlay effects
+   */
+  bgImageUrl?: ConstrainDOMString;
+}
+```
+
+### `createMedia`
+
+```typescript
+import {createMedia, createMediaSignals, UserMediaStatus} from '@pexip/media';
+import type {VideoRenderParams} from '@pexip/media';
+import {
+  urls as mpUrls,
+  createMediapipeSegmenter,
+  createPsySegSegmenter,
+  createCanvasTransform,
+  createVideoTrackProcessor,
+  createVideoTrackProcessorWithFallback,
+  createVideoProcessor,
+  PROCESSING_WIDTH,
+  PROCESSING_HEIGHT,
+  FOREGROUND_THRESHOLD,
+  BACKGROUND_BLUR_AMOUNT,
+  EDGE_BLUR_AMOUNT,
+  FLIP_HORIZONTAL,
+} from '@pexip/media-processor';
+
+// Need some state for the app
+interface State extends VideoRenderParams {
+  mute: {
+    audio: boolean;
+    video: boolean;
+  };
+  noiseSuppression: boolean;
+  vad: boolean;
+  asd: boolean;
+  updateFrequency: number;
+  silentDetectionDuration: number;
+  vadThrottleMS: number;
+  monitor: boolean;
+  channelCount: number;
+}
+export const mediaState: State = {
+  mute: {
+    audio: false,
+    video: false,
+  },
+  videoSegmentation: 'none',
+  noiseSuppression: false,
+  vad: false,
+  asd: false,
+  updateFrequency: 0.5,
+  silentDetectionDuration: 4.0,
+  vadThrottleMS: 3000,
+  width: PROCESSING_WIDTH,
+  height: PROCESSING_HEIGHT,
+  frameRate: 30,
+  bgImageUrl: '',
+  monitor: false,
+  channelCount: 1,
+  foregroundThreshold: FOREGROUND_THRESHOLD,
+  backgroundBlurAmount: BACKGROUND_BLUR_AMOUNT,
+  edgeBlurAmount: EDGE_BLUR_AMOUNT,
+  flipHorizontal: FLIP_HORIZONTAL,
+};
+
+// Create required signals with additional `onDevicesChanged`,
+// `onStatusChanged` and `onStreamTrackEnabled` signals
+export const signals = createMediaSignals('demo', [
+  'onDevicesChanged',
+  'onStatusChanged',
+  'onStreamTrackEnabled',
+]);
+
+const audioProcessor = createAudioStreamProcess({
+  shouldEnable: () => true,
+  denoiseParams: {
+    wasmURL: denoiseWasm.href,
+    workletModule: mpUrls.denoise.href,
+  },
+  analyzerUpdateFrequency: mediaState.updateFrequency,
+  audioSignalDetectionDuration: mediaState.silentDetectionDuration,
+  throttleMs: mediaState.vadThrottleMS,
+  onAudioSignalDetected: signals.onSilentDetected.emit,
+  onVoiceActivityDetected: signals.onVAD.emit,
+});
+
+// Create symbolic link to link bg-blur/selfie_segmentation to a public folder
+// Thus we can create a URL object to get the link to be used for the `gluePath`
+const selfieJs = new URL(
+  `./selfie_segmentation/selfie_segmentation.js`,
+  document.baseURI,
+);
+const mediapipeSegmenter = createMediapipeSegmenter('selfie_segmentation', {
+  modelType: 'general',
+  processingWidth: PROCESSING_WIDTH,
+  processingHeight: PROCESSING_HEIGHT,
+  gluePath: selfieJs.href,
+});
+
+const segmenters: Segmenters = {
+  mediapipeSelfie: mediapipeSegmenter,
+};
+
+const transformer = createCanvasTransform(mediapipeSegmenter, {
+  effects: mediaState.videoSegmentation,
+  width: mediaState.width,
+  height: mediaState.height,
+  foregroundThreshold: mediaState.foregroundThreshold,
+  backgroundBlurAmount: mediaState.backgroundBlurAmount,
+  edgeBlurAmount: mediaState.edgeBlurAmount,
+  flipHorizontal: mediaState.flipHorizontal,
+});
+const videoProcessor = createVideoProcessor([transformer]);
+
+const videoStreamProcessor = createVideoStreamProcess({
+  trackProcessorAPI: 'stream', // Use Stream API when available
+  shouldEnable: () => true, // Always Process Video Track
+  videoSegmentationModel: 'mediapipeSelfie', // Segmentation Modal
+  segmenters,
+  transformer,
+  videoProcessor,
+  processingWidth: mediaState.width,
+  processingHeight: mediaState.height,
+  onError: error => {
+    console.error(error);
+  },
+});
+
+// Instantiate the media object
+export const mediaController = createMedia({
+  getMuteState: () => mediaState.mute,
+  signals,
+  mediaProcessors: [videoStreamProcessor, audioProcessor],
+});
+
+// Hook-up the signals to get the update
+// Subscribe the onMediaChanged signal to get the latest Media object change event
+signals.onMediaChanged.add(media => setLocalMedia(media)); // Assume there is a local function `setLocalMedia`
+// Subscribe the onStatusChanged signal to get the latest Media status change event
+signals.onStatusChanged.add(media => setMediaStatus(media)); // Assume there is a local function `setMediaStatus`
+// Subscribe onStreamTrackEnabled signal to get the MediaStreamTrack['enabled'] change event
+signals.onStreamTrackEnabled.add(track => {
+  console.log(track);
+});
+
+// Later we can initial a gUM to get a MediaStream
+mediaController.getUserMedia({
+  audio: {
+    sampleRate: 48000, // Ideal sample rate for our own noise suppression implementation
+    denoise: true, // Use our own implementation of noise suppression
+    noiseSuppression: false, // Disable the built-in noise suppression
+    vad: true, // Use Voice Activity Detection
+    asd: true, // Use Audio Signal Detection to check if the microphone is malfunctioned
+  },
+  video: {
+    videoSegmentation: 'blur', // Use background blur
+    blurLevel: 21,
+  },
+});
+
+// Access the current `MediaStream`
+mediaController.media.stream;
+// Get audio mute state
+mediaController.media.audioMuted;
+// Mute audio
+mediaController.media.muteAudio(true);
+// Get video mute state
+mediaController.media.videoMuted;
+// Mute video
+mediaController.media.muteVideo(true);
+// Get the status
+mediaController.media.status;
+// Update the status
+mediaController.media.status = UserMediaStatus.Initial;
+// Stop the `MediaStream`
+await mediaController.media.release();
+
+// Later we can make changes to the media on-demand without initiate a new gUM call
+
+// Turn off background blur
+await mediaController.media.applyConstraints({
+  video: {videoSegmentation: 'none'},
+});
+// Turn on background blur
+await mediaController.media.applyConstraints({
+  video: {videoSegmentation: 'blur'},
+});
+// Use our own noise suppression
+await mediaController.media.applyConstraints({
+  audio: {denoise: true, noiseSuppression: false},
+});
+// Use build-in noise suppression
+await mediaController.media.applyConstraints({
+  audio: {denoise: false, noiseSuppression: true},
+});
+```
+
+### `createPreviewController`
+
+```typescript
+const {createPreviewController} = require('@pexip/media');
+
+// Some states from the App
+interface AppState {
+  media: Media;
+  devices: MediaDeviceInfo[];
+}
+// Some functions from the App
+const updateStream = (constraints: MediaDeviceRequest) => Promise<void>;
+
+const onMediaChangedSignal = createSignal<Media>();
+
+// Instantiate the controller
+const controller = createPreviewController({
+  getDesiredResolution: () => ({}),
+  getCurrentDevices: () => state.devices,
+  getCurrentMedia: () => state.media,
+  updateMainStream: updateStream,
+  mediaSignal: onMediaChangedSignal,
+  getSegmentationOptions: () => ({
+    effects: mediaState.videoSegmentation,
+    erode: mediaState.erode,
+    blurLevel: mediaState.blurLevel,
+  }),
+  videoProcessingEnabled: true,
+  updateMainStream: mc.getUserMediaAsync,
+  onEnded: () => {
+    close();
+  },
+  psySegScriptLoaded: true,
+});
+
+// Want to learn more? Please checkout the `@pexip/media-demo` sample app to try
+```