@stream-io/video-react-sdk 1.2.12 → 1.2.13

package/CHANGELOG.md

@@ -2,6 +2,19 @@
 
 This file was generated using [@jscutlery/semver](https://github.com/jscutlery/semver).
 
+### [1.2.13](https://github.com/GetStream/stream-video-js/compare/@stream-io/video-react-sdk-1.2.12...@stream-io/video-react-sdk-1.2.13) (2024-07-02)
+
+### Dependency Updates
+
+* `@stream-io/video-client` updated to version `1.4.4`
+* `@stream-io/video-filters-web` updated to version `0.1.3`
+* `@stream-io/video-react-bindings` updated to version `0.4.48`
+* `@stream-io/audio-filters-web` updated to version `0.2.1`
+
+### Bug Fixes
+
+* refactor background filters ([#1415](https://github.com/GetStream/stream-video-js/issues/1415)) ([deb6da2](https://github.com/GetStream/stream-video-js/commit/deb6da238f541c733451e84b198434671da8dceb))
+
 ### [1.2.12](https://github.com/GetStream/stream-video-js/compare/@stream-io/video-react-sdk-1.2.11...@stream-io/video-react-sdk-1.2.12) (2024-07-02)
 
 

package/dist/index.cjs.js

@@ -5,6 +5,7 @@ var videoReactBindings = require('@stream-io/video-react-bindings');
 var jsxRuntime = require('react/jsx-runtime');
 var react = require('react');
 var clsx = require('clsx');
+var reactDom = require('react-dom');
 var videoFiltersWeb = require('@stream-io/video-filters-web');
 var react$1 = require('@floating-ui/react');
 var chart_js = require('chart.js');
@@ -74,7 +75,7 @@ const useBackgroundFilters = () => {
  * in your project before using this component.
  */
 const BackgroundFiltersProvider = (props) => {
-    const { children, backgroundImages = [], backgroundFilter: bgFilterFromProps = undefined, backgroundImage: bgImageFromProps = undefined, backgroundBlurLevel: bgBlurLevelFromProps = 'high', tfFilePath, modelFilePath, basePath, } = props;
+    const { children, backgroundImages = [], backgroundFilter: bgFilterFromProps = undefined, backgroundImage: bgImageFromProps = undefined, backgroundBlurLevel: bgBlurLevelFromProps = 'high', tfFilePath, modelFilePath, basePath, onError, } = props;
     const [backgroundFilter, setBackgroundFilter] = react.useState(bgFilterFromProps);
     const [backgroundImage, setBackgroundImage] = react.useState(bgImageFromProps);
     const [backgroundBlurLevel, setBackgroundBlurLevel] = react.useState(bgBlurLevelFromProps);
@@ -104,6 +105,11 @@ const BackgroundFiltersProvider = (props) => {
             .then(setTfLite)
             .catch((err) => console.error('Failed to load TFLite', err));
     }, [basePath, isSupported, modelFilePath, tfFilePath]);
+    const handleError = react.useCallback((error) => {
+        videoClient.getLogger(['filters'])('warn', 'Filter encountered an error and will be disabled');
+        disableBackgroundFilter();
+        onError?.(error);
+    }, [disableBackgroundFilter, onError]);
     return (jsxRuntime.jsxs(BackgroundFiltersContext.Provider, { value: {
             isSupported,
             isReady: !!tfLite,
@@ -117,104 +123,88 @@ const BackgroundFiltersProvider = (props) => {
             tfFilePath,
             modelFilePath,
             basePath,
-        }, children: [children, tfLite && backgroundFilter && jsxRuntime.jsx(BackgroundFilters, { tfLite: tfLite })] }));
+            onError: handleError,
+        }, children: [children, tfLite && jsxRuntime.jsx(BackgroundFilters, { tfLite: tfLite })] }));
 };
 const BackgroundFilters = (props) => {
-    const { tfLite } = props;
     const call = videoReactBindings.useCall();
-    const { backgroundImage, backgroundFilter } = useBackgroundFilters();
-    const [videoRef, setVideoRef] = react.useState(null);
-    const [bgImageRef, setBgImageRef] = react.useState(null);
-    const [canvasRef, setCanvasRef] = react.useState(null);
-    const [width, setWidth] = react.useState(1920);
-    const [height, setHeight] = react.useState(1080);
-    // Holds a ref to the `resolve` function of the returned Promise as part
-    // of the `camera.registerFilter()` API. Once the filter is initialized,
-    // it should be called with the filtered MediaStream as an argument.
-    const signalFilterReadyRef = react.useRef();
-    const [mediaStream, setMediaStream] = react.useState();
-    const unregister = react.useRef();
+    const { children, start } = useRenderer(props.tfLite);
+    const { backgroundFilter, onError } = useBackgroundFilters();
+    const handleErrorRef = react.useRef(undefined);
+    handleErrorRef.current = onError;
     react.useEffect(() => {
         if (!call || !backgroundFilter)
             return;
-        const register = (unregister.current || Promise.resolve()).then(() => call.camera.registerFilter(async (ms) => {
-            return new Promise((resolve) => {
-                signalFilterReadyRef.current = resolve;
-                setMediaStream(ms);
-            });
-        }));
+        const { unregister } = call.camera.registerFilter((ms) => start(ms, (error) => handleErrorRef.current?.(error)));
         return () => {
-            unregister.current = register
-                .then((unregisterFilter) => unregisterFilter())
-                .then(() => (signalFilterReadyRef.current = undefined))
-                .then(() => setMediaStream(undefined))
-                .catch((err) => console.error('Failed to unregister filter', err));
+            unregister();
         };
-    }, [backgroundFilter, call]);
-    const [isPlaying, setIsPlaying] = react.useState(false);
-    react.useEffect(() => {
-        if (!mediaStream || !videoRef)
-            return;
-        const handleOnPlay = () => {
-            const [track] = mediaStream.getVideoTracks();
-            if (!track)
+    }, [backgroundFilter, call, start]);
+    return children;
+};
+const useRenderer = (tfLite) => {
+    const { backgroundFilter, backgroundBlurLevel, backgroundImage } = useBackgroundFilters();
+    const videoRef = react.useRef(null);
+    const canvasRef = react.useRef(null);
+    const bgImageRef = react.useRef(null);
+    const [videoSize, setVideoSize] = react.useState({
+        width: 1920,
+        height: 1080,
+    });
+    const start = react.useCallback((ms, onError) => {
+        let outputStream;
+        let renderer;
+        const output = new Promise((resolve, reject) => {
+            if (!backgroundFilter) {
+                reject(new Error('No filter specified'));
                 return;
-            const { width: w = 0, height: h = 0 } = track.getSettings();
-            setWidth(w);
-            setHeight(h);
-            setIsPlaying(true);
-        };
-        videoRef.addEventListener('play', handleOnPlay);
-        videoRef.srcObject = mediaStream;
-        videoRef.play().catch((err) => {
-            console.error('Failed to play video', err);
-        });
-        return () => {
-            videoRef.removeEventListener('play', handleOnPlay);
-            videoRef.srcObject = null;
-            setIsPlaying(false);
-        };
-    }, [mediaStream, videoRef]);
-    react.useEffect(() => {
-        const resolveFilter = signalFilterReadyRef.current;
-        if (!canvasRef || !resolveFilter)
-            return;
-        const filter = canvasRef.captureStream();
-        resolveFilter(filter);
-        return () => {
-            videoClient.disposeOfMediaStream(filter);
-        };
-    }, [canvasRef]);
-    return (jsxRuntime.jsxs("div", { className: "str-video__background-filters", style: {
-            width: `${width}px`,
-            height: `${height}px`,
-        }, children: [mediaStream && isPlaying && (jsxRuntime.jsx(RenderPipeline, { tfLite: tfLite, videoRef: videoRef, canvasRef: canvasRef, backgroundImageRef: bgImageRef })), jsxRuntime.jsx("video", { className: clsx('str-video__background-filters__video', height > width && 'str-video__background-filters__video--tall'), ref: setVideoRef, autoPlay: true, playsInline: true, controls: false, width: width, height: height, muted: true, loop: true }), backgroundImage && (jsxRuntime.jsx("img", { className: "str-video__background-filters__background-image", alt: "Background", ref: setBgImageRef, src: backgroundImage, width: width, height: height }, backgroundImage)), isPlaying && (jsxRuntime.jsx("canvas", { className: "str-video__background-filters__target-canvas", width: width, height: height, ref: setCanvasRef }))] }));
-};
-const RenderPipeline = (props) => {
-    const { tfLite, videoRef, canvasRef, backgroundImageRef } = props;
-    const { backgroundFilter, backgroundBlurLevel } = useBackgroundFilters();
-    react.useEffect(() => {
-        if (!videoRef || !canvasRef || !backgroundFilter)
-            return;
-        if (backgroundFilter === 'image' && !backgroundImageRef)
-            return;
-        const renderer = videoFiltersWeb.createRenderer(tfLite, videoRef, canvasRef, {
-            backgroundFilter,
-            backgroundImage: backgroundImageRef ?? undefined,
-            backgroundBlurLevel,
+            }
+            const videoEl = videoRef.current;
+            const canvasEl = canvasRef.current;
+            const bgImageEl = bgImageRef.current;
+            if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
+                // You should start renderer in effect or event handlers
+                reject(new Error('Renderer started before elements are ready'));
+                return;
+            }
+            videoEl.srcObject = ms;
+            videoEl.play().then(() => {
+                const [track] = ms.getVideoTracks();
+                if (!track) {
+                    reject(new Error('No video tracks in input media stream'));
+                    return;
+                }
+                const trackSettings = track.getSettings();
+                reactDom.flushSync(() => setVideoSize({
+                    width: trackSettings.width ?? 0,
+                    height: trackSettings.height ?? 0,
+                }));
+                renderer = videoFiltersWeb.createRenderer(tfLite, videoEl, canvasEl, {
+                    backgroundFilter,
+                    backgroundBlurLevel,
+                    backgroundImage: bgImageEl ?? undefined,
+                }, onError);
+                outputStream = canvasEl.captureStream();
+                resolve(outputStream);
+            }, () => {
+                reject(new Error('Could not play the source video stream'));
+            });
         });
-        return () => {
-            renderer.dispose();
+        return {
+            output,
+            stop: () => {
+                renderer?.dispose();
+                videoRef.current && (videoRef.current.srcObject = null);
+                outputStream && videoClient.disposeOfMediaStream(outputStream);
+            },
         };
-    }, [
-        backgroundBlurLevel,
-        backgroundFilter,
-        backgroundImageRef,
-        canvasRef,
-        tfLite,
-        videoRef,
-    ]);
-    return null;
+    }, [backgroundBlurLevel, backgroundFilter, backgroundImage, tfLite]);
+    const children = (jsxRuntime.jsxs("div", { className: "str-video__background-filters", children: [jsxRuntime.jsx("video", { className: clsx('str-video__background-filters__video', videoSize.height > videoSize.width &&
+                    'str-video__background-filters__video--tall'), ref: videoRef, playsInline: true, muted: true, controls: false, ...videoSize }), backgroundImage && (jsxRuntime.jsx("img", { className: "str-video__background-filters__background-image", alt: "Background", ref: bgImageRef, src: backgroundImage, ...videoSize })), jsxRuntime.jsx("canvas", { className: "str-video__background-filters__target-canvas", ...videoSize, ref: canvasRef })] }));
+    return {
+        start,
+        children,
+    };
 };
 
 const useFloatingUIPreset = ({ middleware = [], placement, strategy, offset: offsetInPx = 10, }) => {
@@ -2622,7 +2612,7 @@ const LivestreamPlayer = (props) => {
     return (jsxRuntime.jsx(StreamCall, { call: call, children: jsxRuntime.jsx(LivestreamLayout, { ...layoutProps }) }));
 };
 
-const [major, minor, patch] = ("1.2.12" ).split('.');
+const [major, minor, patch] = ("1.2.13" ).split('.');
 videoClient.setSdkInfo({
     type: videoClient.SfuModels.SdkType.REACT,
     major,