@pexip/media 22.1.0 → 22.2.0

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @pexip/media
 
+## 22.2.0
+
+### Patch Changes
+
+- 72f8dde: Temporarily remove updating the backend in the pipeline
+- 746f6b6: Add Canvas2d rendering backend support: alpha mask converter,
+  Canvas2d mask renderer, canvas dimension-aware rendering, and proper backend
+  propagation through the video processor pipeline.
+- 265c898: Migrate test runner from Jest to Vitest.
+- a5e8026: Fix "We can't find your microphone" error when changing the
+  microphone while sharing content without audio. The audio mixing processor no
+  longer tries to create an audio source node from a presentation stream that
+  has no audio track, and audio nodes belonging to a stale `AudioContext` are
+  recreated instead of being reused.
+- 0878b3b: Fix race conditions in the video processing pipeline so effect
+  changes made during processor startup are correctly applied and worker option
+  diffs are computed against the previous state.
+- Updated dependencies [5a17991]
+- Updated dependencies [f4af45a]
+- Updated dependencies [746f6b6]
+- Updated dependencies [c101d02]
+- Updated dependencies [476acbf]
+- Updated dependencies [7c99e29]
+- Updated dependencies [1b82fd5]
+- Updated dependencies [265c898]
+- Updated dependencies [a5e8026]
+- Updated dependencies [0878b3b]
+  - @pexip/utils@17.4.1
+  - @pexip/media-processor@22.2.0
+  - @pexip/media-control@22.2.0
+  - @pexip/signal@16.9.6
+
 ## 22.1.0
 
 ### Minor Changes

package/dist/audioMixingProcessor.d.ts

@@ -5,6 +5,7 @@ interface AudioStreamProcessorProps {
     mixWithAdditionalMedia?: boolean;
     merger?: AudioNodeInit<ChannelMergerNode, ChannelMergerNode>;
     displaySource?: AudioNodeInit<MediaStreamAudioSourceNode, MediaStreamAudioSourceNode>;
+    displayStream?: MediaStream;
     audioGraph?: AudioGraph;
 }
 declare const FEATURE_KEYS: {
@@ -13,6 +14,20 @@ declare const FEATURE_KEYS: {
 type FeaturePropKeys = keyof typeof FEATURE_KEYS;
 type FeatureProps = Pick<AudioStreamProcessorProps, FeaturePropKeys>;
 export declare const updateFeatureProps: (constraints: MediaDeviceRequest["audio"], props: FeatureProps) => FeatureProps;
+/**
+ * Extract a `MediaStream` containing only the audio tracks of the provided
+ * stream, or `undefined` when there is no audio track to mix, e.g. sharing a
+ * screen without audio.
+ *
+ * @param stream - The stream to extract the audio tracks from
+ *
+ * @example
+ *
+ * ```typescript
+ * const audioOnly = toAudioOnlyStream(displayStream);
+ * ```
+ */
+export declare const toAudioOnlyStream: (stream?: MediaStream) => MediaStream | undefined;
 /**
  * Create an Audio Mixing Processor and will own the stream passed-in
  */

package/dist/audioMixingProcessor.js

@@ -28,6 +28,26 @@ export const updateFeatureProps = (constraints, props) => {
     }
     return features;
 };
+/**
+ * Extract a `MediaStream` containing only the audio tracks of the provided
+ * stream, or `undefined` when there is no audio track to mix, e.g. sharing a
+ * screen without audio.
+ *
+ * @param stream - The stream to extract the audio tracks from
+ *
+ * @example
+ *
+ * ```typescript
+ * const audioOnly = toAudioOnlyStream(displayStream);
+ * ```
+ */
+export const toAudioOnlyStream = (stream) => {
+    const audioTracks = stream?.getAudioTracks?.() ?? [];
+    if (audioTracks.length === 0) {
+        return undefined;
+    }
+    return new MediaStream(audioTracks);
+};
 /**
  * Create an Audio Mixing Processor and will own the stream passed-in
  */
@@ -36,7 +56,8 @@ export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCE
     return async (prevMediaTrack) => {
         updateFeatureProps(prevMediaTrack.getConstraints(), props);
         const displayStream = getCurrentMedia();
-        const displayTrack = displayStream?.getAudioTracks?.().at(0);
+        const displayAudioStream = toAudioOnlyStream(displayStream);
+        const displayTrack = displayAudioStream?.getAudioTracks().at(0);
         const applyConstraints = async (constraints) => {
             if (isEmpty(constraints)) {
                 return;
@@ -50,8 +71,8 @@ export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCE
             }
             if (features.mixWithAdditionalMedia) {
                 const newStream = getCurrentMedia();
-                const [newTrack] = getCurrentMedia()?.getAudioTracks() ?? [];
-                if (!newTrack) {
+                const newAudioStream = toAudioOnlyStream(newStream);
+                if (!newAudioStream) {
                     return;
                 }
                 if (!props.audioGraph || !props.merger) {
@@ -59,8 +80,7 @@ export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCE
                     return;
                 }
                 if (props.displaySource && props.merger) {
-                    if (newStream ===
-                        props.displaySource.audioNode?.mediaStream) {
+                    if (newStream === props.displayStream) {
                         return;
                     }
                     props.audioGraph.disconnect([
@@ -68,14 +88,13 @@ export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCE
                         props.merger,
                     ]);
                 }
-                if (newStream) {
-                    props.displaySource =
-                        createStreamSourceGraphNode(newStream);
-                    props.audioGraph.connect([
-                        props.displaySource,
-                        props.merger,
-                    ]);
-                }
+                props.displayStream = newStream;
+                props.displaySource =
+                    createStreamSourceGraphNode(newAudioStream);
+                props.audioGraph.connect([
+                    props.displaySource,
+                    props.merger,
+                ]);
             }
             else if (features.mixWithAdditionalMedia === false) {
                 // Unmixing
@@ -83,6 +102,7 @@ export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCE
                     props.displaySource.release();
                     props.audioGraph?.disconnect([props.displaySource]);
                     props.displaySource = undefined;
+                    props.displayStream = undefined;
                 }
             }
             return Promise.resolve();
@@ -118,8 +138,10 @@ export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCE
         }
         assert(prevMediaTrack.track, 'Main track should be available');
         const mainSource = createStreamSourceGraphNode(new MediaStream([prevMediaTrack.track]));
-        props.displaySource =
-            displayStream && createStreamSourceGraphNode(displayStream);
+        props.displaySource = displayAudioStream
+            ? createStreamSourceGraphNode(displayAudioStream)
+            : undefined;
+        props.displayStream = displayAudioStream ? displayStream : undefined;
         props.merger = createChannelMergerGraphNode();
         const destination = createStreamDestinationGraphNode({
             channelCount: 1,
@@ -151,10 +173,13 @@ export const createAudioMixingProcess = (getCurrentMedia, signals, label = PROCE
             logger.debug({ label }, 'Release Media');
             unsubscribe();
             // Release Props
-            await props.audioGraph?.release();
-            props.audioGraph = undefined;
-            props.merger = undefined;
-            props.displaySource = undefined;
+            await audioGraph.release();
+            if (props.audioGraph === audioGraph) {
+                props.audioGraph = undefined;
+                props.merger = undefined;
+                props.displaySource = undefined;
+                props.displayStream = undefined;
+            }
         };
         return Promise.resolve(createMediaTrack({
             label,

package/dist/audioProcessor.js

@@ -173,6 +173,14 @@ label = PROCESSOR_LABELS.AudioProcessor, }) => {
             [source, analyzer],
         ];
         logger.debug({ initialAudioNodeConnection, label }, 'Initial AudioNodeConnection');
+        if (props.denoiseNode) {
+            props.denoiseNode.release();
+            props.denoiseNode = undefined;
+        }
+        if (props.analyzer) {
+            props.analyzer.release();
+            props.analyzer = undefined;
+        }
         const audioGraph = createAudioGraphProxy(createAudioGraph(initialAudioNodeConnection, props.audioGraphOptions), {
             connect: (target, args) => {
                 logger.debug({ label, target, args }, 'connect nodes');
@@ -199,9 +207,11 @@ label = PROCESSOR_LABELS.AudioProcessor, }) => {
             track.stop();
             // Release Props
             await audioGraph.release();
-            props.denoiseNode = undefined;
-            props.analyzer = undefined;
-            props.audioGraph = undefined;
+            if (props.audioGraph === audioGraph) {
+                props.denoiseNode = undefined;
+                props.analyzer = undefined;
+                props.audioGraph = undefined;
+            }
         };
         return createMediaTrack({
             label,

package/dist/videoProcessor.d.ts

@@ -22,6 +22,7 @@ interface VideoStreamProcessProps extends Partial<VideoRenderParams> {
     contentHint?: VideoContentHint;
 }
 declare const FEATURE_KEYS: {
+    readonly backend: "string";
     readonly backgroundBlurAmount: "number";
     readonly backgroundImageUrl: "string";
     readonly backgroundThreshold: "number";

package/dist/videoProcessor.js

@@ -6,6 +6,7 @@ import { logger } from './logger';
 import { isVideoContentHint } from './typeGuard';
 import { PROCESSOR_LABELS } from './constants';
 const FEATURE_KEYS = {
+    backend: 'string',
     backgroundBlurAmount: 'number',
     backgroundImageUrl: 'string',
     backgroundThreshold: 'number',
@@ -217,21 +218,26 @@ const applyFeatures = async (processor, features, referenceProcessingHeight) =>
             case 'downSampleFactor':
             case 'backgroundThreshold': {
                 const value = features[k];
-                if (value !== undefined) {
+                if (value !== undefined &&
+                    value !== processor.renderOptions[k]) {
                     renderOptions[k] = value;
                 }
                 break;
             }
             case 'excludeBystanders': {
                 const value = features[k];
-                if (value !== undefined) {
+                if (value !== undefined &&
+                    value !== processor.renderOptions[k]) {
                     renderOptions[k] = value;
                 }
                 break;
             }
             case 'personCenterX': {
                 const value = features[k];
-                if (value !== undefined) {
+                if (value !== undefined &&
+                    value !== processor.renderOptions.personCenter[0]) {
+                    // Keep the other coordinate from the local draft or existing
+                    // processor state so X/Y single-field updates stay paired.
                     renderOptions.personCenter = [
                         value,
                         renderOptions.personCenter?.[1] ??
@@ -242,7 +248,10 @@ const applyFeatures = async (processor, features, referenceProcessingHeight) =>
             }
             case 'personCenterY': {
                 const value = features[k];
-                if (value !== undefined) {
+                if (value !== undefined &&
+                    value !== processor.renderOptions.personCenter[1]) {
+                    // Keep the other coordinate from the local draft or existing
+                    // processor state so X/Y single-field updates stay paired.
                     renderOptions.personCenter = [
                         renderOptions.personCenter?.[0] ??
                             processor.renderOptions.personCenter[0],
@@ -255,14 +264,20 @@ const applyFeatures = async (processor, features, referenceProcessingHeight) =>
             case 'backgroundBlurAmount': {
                 const value = features[k];
                 if (value !== undefined) {
-                    renderOptions[k] = getBlurKernelSize(value, referenceProcessingHeight ?? processor.processingHeight, processor.processingHeight);
+                    const actualValue = getBlurKernelSize(value, referenceProcessingHeight ?? processor.processingHeight, processor.processingHeight);
+                    if (actualValue !== processor.renderOptions[k]) {
+                        renderOptions[k] = actualValue;
+                    }
                 }
                 break;
             }
             case 'edgeBlurAmount': {
                 const value = features[k];
                 if (value !== undefined) {
-                    renderOptions[k] = getTentBlurSize(value, referenceProcessingHeight ?? processor.processingHeight, processor.processingHeight);
+                    const actualValue = getTentBlurSize(value, referenceProcessingHeight ?? processor.processingHeight, processor.processingHeight);
+                    if (actualValue !== processor.renderOptions[k]) {
+                        renderOptions[k] = actualValue;
+                    }
                 }
                 break;
             }
@@ -273,6 +288,13 @@ const applyFeatures = async (processor, features, referenceProcessingHeight) =>
                 }
                 break;
             }
+            case 'backend': {
+                const value = features[k];
+                if (value && value !== processor.renderOptions.backend) {
+                    renderOptions.backend = value;
+                }
+                break;
+            }
             case 'backgroundImageUrl': {
                 const value = features[k];
                 if (value &&
@@ -356,9 +378,23 @@ label = PROCESSOR_LABELS.VideoProcessor, referenceProcessingHeight, maskCombineR
             });
         }
         else {
-            await applyFeatures(videoProcessor, { ...features, videoSegmentation }, referenceProcessingHeight);
+            await applyFeatures(videoProcessor, {
+                backend: gpuAPI(),
+                ...renderOptions,
+                videoSegmentation,
+            }, referenceProcessingHeight);
         }
         const track = await videoProcessor.process(prevMediaTrack.track);
+        // The source track constraints may have changed while `process()` was
+        // awaiting the segmentation model to load (e.g. the user toggled blur
+        // in pre-flight before the processor was ready). The features computed
+        // above are already baked into the freshly opened processor, so
+        // re-apply anything that changed in the meantime to avoid silently
+        // dropping the effect.
+        const latestFeatures = updateFeatureProps(prevMediaTrack.getConstraints(), props);
+        if (!isEmpty(latestFeatures)) {
+            await applyFeatures(videoProcessor, latestFeatures, referenceProcessingHeight);
+        }
         // Inherit contentHint from previous track
         track.contentHint = prevMediaTrack.track.contentHint;
         let trackReleaseTimeoutID = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pexip/media",
-  "version": "22.1.0",
+  "version": "22.2.0",
   "description": "Home for media related stuff",
   "homepage": "https://developer.pexip.com",
   "bugs": "https://gitlab.com/pexip/zoo/issues",
@@ -39,7 +39,7 @@
     "clean": "rm -fr dist api-docs",
     "dev": "pexip-bundler dev",
     "prepack": "yarn clean && yarn build && yarn docs",
-    "test": "jest",
+    "test": "pexip-bundler test",
     "check-format": "prettier --ignore-path=../../.prettierignore --check .",
     "format": "prettier --ignore-path=../../.prettierignore --write .",
     "lint": "yarn run -T biome lint",
@@ -47,18 +47,13 @@
     "docs": "typedoc"
   },
   "dependencies": {
-    "@pexip/media-control": "22.1.0",
-    "@pexip/media-processor": "22.1.0",
-    "@pexip/signal": "16.9.5",
-    "@pexip/utils": "17.4.0"
+    "@pexip/media-control": "22.2.0",
+    "@pexip/media-processor": "22.2.0",
+    "@pexip/signal": "16.9.6",
+    "@pexip/utils": "17.4.1"
   },
   "devDependencies": {
-    "@jest/globals": "^29.5.12",
-    "@pexip/bundler": "18.2.1",
-    "@swc/core": "^1.15.33",
-    "@swc/jest": "^0.2.39",
-    "jest": "^29.5.12",
-    "jest-junit": "^16.0.0",
+    "@pexip/bundler": "18.3.0",
     "prettier": "^3.2.5",
     "typedoc": "^0.28.18",
     "typedoc-plugin-markdown": "^4.11.0",