@stream-io/video-react-sdk 1.2.12 → 1.2.13

package/dist/index.es.js

@@ -1,10 +1,11 @@
-import { hasAudio, hasScreenShareAudio, disposeOfMediaStream, CallingState, OwnCapability, getLogger, hasVideo, isPinned, name, NoiseCancellationSettingsModeEnum, hasScreenShare, VisibilityState, Browsers, SfuModels, paginatedLayoutSortPreset, combineComparators, screenSharing, speakerLayoutSortPreset, CallTypes, defaultSortPreset, setSdkInfo } from '@stream-io/video-client';
+import { hasAudio, hasScreenShareAudio, getLogger, disposeOfMediaStream, CallingState, OwnCapability, hasVideo, isPinned, name, NoiseCancellationSettingsModeEnum, hasScreenShare, VisibilityState, Browsers, SfuModels, paginatedLayoutSortPreset, combineComparators, screenSharing, speakerLayoutSortPreset, CallTypes, defaultSortPreset, setSdkInfo } from '@stream-io/video-client';
 export * from '@stream-io/video-client';
 import { useCall, useCallStateHooks, useI18n, Restricted, useConnectedUser, StreamCallProvider, StreamVideoProvider, useStreamVideoClient } from '@stream-io/video-react-bindings';
 export * from '@stream-io/video-react-bindings';
 import { jsx, Fragment, jsxs } from 'react/jsx-runtime';
 import { useState, useEffect, Fragment as Fragment$1, createContext, useContext, useCallback, useRef, useMemo, forwardRef, isValidElement, useLayoutEffect } from 'react';
 import clsx from 'clsx';
+import { flushSync } from 'react-dom';
 import { isPlatformSupported, loadTFLite, createRenderer } from '@stream-io/video-filters-web';
 import { useFloating, offset, shift, flip, size, autoUpdate, FloatingOverlay, FloatingPortal, arrow, FloatingArrow, useListItem, useListNavigation, useTypeahead, useClick, useDismiss, useRole, useInteractions, FloatingFocusManager, FloatingList, useHover } from '@floating-ui/react';
 import { Chart, CategoryScale, LinearScale, LineElement, PointElement } from 'chart.js';
@@ -74,7 +75,7 @@ const useBackgroundFilters = () => {
  * in your project before using this component.
  */
 const BackgroundFiltersProvider = (props) => {
-    const { children, backgroundImages = [], backgroundFilter: bgFilterFromProps = undefined, backgroundImage: bgImageFromProps = undefined, backgroundBlurLevel: bgBlurLevelFromProps = 'high', tfFilePath, modelFilePath, basePath, } = props;
+    const { children, backgroundImages = [], backgroundFilter: bgFilterFromProps = undefined, backgroundImage: bgImageFromProps = undefined, backgroundBlurLevel: bgBlurLevelFromProps = 'high', tfFilePath, modelFilePath, basePath, onError, } = props;
     const [backgroundFilter, setBackgroundFilter] = useState(bgFilterFromProps);
     const [backgroundImage, setBackgroundImage] = useState(bgImageFromProps);
     const [backgroundBlurLevel, setBackgroundBlurLevel] = useState(bgBlurLevelFromProps);
@@ -104,6 +105,11 @@ const BackgroundFiltersProvider = (props) => {
             .then(setTfLite)
             .catch((err) => console.error('Failed to load TFLite', err));
     }, [basePath, isSupported, modelFilePath, tfFilePath]);
+    const handleError = useCallback((error) => {
+        getLogger(['filters'])('warn', 'Filter encountered an error and will be disabled');
+        disableBackgroundFilter();
+        onError?.(error);
+    }, [disableBackgroundFilter, onError]);
     return (jsxs(BackgroundFiltersContext.Provider, { value: {
             isSupported,
             isReady: !!tfLite,
@@ -117,104 +123,88 @@ const BackgroundFiltersProvider = (props) => {
             tfFilePath,
             modelFilePath,
             basePath,
-        }, children: [children, tfLite && backgroundFilter && jsx(BackgroundFilters, { tfLite: tfLite })] }));
+            onError: handleError,
+        }, children: [children, tfLite && jsx(BackgroundFilters, { tfLite: tfLite })] }));
 };
 const BackgroundFilters = (props) => {
-    const { tfLite } = props;
     const call = useCall();
-    const { backgroundImage, backgroundFilter } = useBackgroundFilters();
-    const [videoRef, setVideoRef] = useState(null);
-    const [bgImageRef, setBgImageRef] = useState(null);
-    const [canvasRef, setCanvasRef] = useState(null);
-    const [width, setWidth] = useState(1920);
-    const [height, setHeight] = useState(1080);
-    // Holds a ref to the `resolve` function of the returned Promise as part
-    // of the `camera.registerFilter()` API. Once the filter is initialized,
-    // it should be called with the filtered MediaStream as an argument.
-    const signalFilterReadyRef = useRef();
-    const [mediaStream, setMediaStream] = useState();
-    const unregister = useRef();
+    const { children, start } = useRenderer(props.tfLite);
+    const { backgroundFilter, onError } = useBackgroundFilters();
+    const handleErrorRef = useRef(undefined);
+    handleErrorRef.current = onError;
     useEffect(() => {
         if (!call || !backgroundFilter)
             return;
-        const register = (unregister.current || Promise.resolve()).then(() => call.camera.registerFilter(async (ms) => {
-            return new Promise((resolve) => {
-                signalFilterReadyRef.current = resolve;
-                setMediaStream(ms);
-            });
-        }));
+        const { unregister } = call.camera.registerFilter((ms) => start(ms, (error) => handleErrorRef.current?.(error)));
         return () => {
-            unregister.current = register
-                .then((unregisterFilter) => unregisterFilter())
-                .then(() => (signalFilterReadyRef.current = undefined))
-                .then(() => setMediaStream(undefined))
-                .catch((err) => console.error('Failed to unregister filter', err));
+            unregister();
         };
-    }, [backgroundFilter, call]);
-    const [isPlaying, setIsPlaying] = useState(false);
-    useEffect(() => {
-        if (!mediaStream || !videoRef)
-            return;
-        const handleOnPlay = () => {
-            const [track] = mediaStream.getVideoTracks();
-            if (!track)
+    }, [backgroundFilter, call, start]);
+    return children;
+};
+const useRenderer = (tfLite) => {
+    const { backgroundFilter, backgroundBlurLevel, backgroundImage } = useBackgroundFilters();
+    const videoRef = useRef(null);
+    const canvasRef = useRef(null);
+    const bgImageRef = useRef(null);
+    const [videoSize, setVideoSize] = useState({
+        width: 1920,
+        height: 1080,
+    });
+    const start = useCallback((ms, onError) => {
+        let outputStream;
+        let renderer;
+        const output = new Promise((resolve, reject) => {
+            if (!backgroundFilter) {
+                reject(new Error('No filter specified'));
                 return;
-            const { width: w = 0, height: h = 0 } = track.getSettings();
-            setWidth(w);
-            setHeight(h);
-            setIsPlaying(true);
-        };
-        videoRef.addEventListener('play', handleOnPlay);
-        videoRef.srcObject = mediaStream;
-        videoRef.play().catch((err) => {
-            console.error('Failed to play video', err);
-        });
-        return () => {
-            videoRef.removeEventListener('play', handleOnPlay);
-            videoRef.srcObject = null;
-            setIsPlaying(false);
-        };
-    }, [mediaStream, videoRef]);
-    useEffect(() => {
-        const resolveFilter = signalFilterReadyRef.current;
-        if (!canvasRef || !resolveFilter)
-            return;
-        const filter = canvasRef.captureStream();
-        resolveFilter(filter);
-        return () => {
-            disposeOfMediaStream(filter);
-        };
-    }, [canvasRef]);
-    return (jsxs("div", { className: "str-video__background-filters", style: {
-            width: `${width}px`,
-            height: `${height}px`,
-        }, children: [mediaStream && isPlaying && (jsx(RenderPipeline, { tfLite: tfLite, videoRef: videoRef, canvasRef: canvasRef, backgroundImageRef: bgImageRef })), jsx("video", { className: clsx('str-video__background-filters__video', height > width && 'str-video__background-filters__video--tall'), ref: setVideoRef, autoPlay: true, playsInline: true, controls: false, width: width, height: height, muted: true, loop: true }), backgroundImage && (jsx("img", { className: "str-video__background-filters__background-image", alt: "Background", ref: setBgImageRef, src: backgroundImage, width: width, height: height }, backgroundImage)), isPlaying && (jsx("canvas", { className: "str-video__background-filters__target-canvas", width: width, height: height, ref: setCanvasRef }))] }));
-};
-const RenderPipeline = (props) => {
-    const { tfLite, videoRef, canvasRef, backgroundImageRef } = props;
-    const { backgroundFilter, backgroundBlurLevel } = useBackgroundFilters();
-    useEffect(() => {
-        if (!videoRef || !canvasRef || !backgroundFilter)
-            return;
-        if (backgroundFilter === 'image' && !backgroundImageRef)
-            return;
-        const renderer = createRenderer(tfLite, videoRef, canvasRef, {
-            backgroundFilter,
-            backgroundImage: backgroundImageRef ?? undefined,
-            backgroundBlurLevel,
+            }
+            const videoEl = videoRef.current;
+            const canvasEl = canvasRef.current;
+            const bgImageEl = bgImageRef.current;
+            if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
+                // You should start renderer in effect or event handlers
+                reject(new Error('Renderer started before elements are ready'));
+                return;
+            }
+            videoEl.srcObject = ms;
+            videoEl.play().then(() => {
+                const [track] = ms.getVideoTracks();
+                if (!track) {
+                    reject(new Error('No video tracks in input media stream'));
+                    return;
+                }
+                const trackSettings = track.getSettings();
+                flushSync(() => setVideoSize({
+                    width: trackSettings.width ?? 0,
+                    height: trackSettings.height ?? 0,
+                }));
+                renderer = createRenderer(tfLite, videoEl, canvasEl, {
+                    backgroundFilter,
+                    backgroundBlurLevel,
+                    backgroundImage: bgImageEl ?? undefined,
+                }, onError);
+                outputStream = canvasEl.captureStream();
+                resolve(outputStream);
+            }, () => {
+                reject(new Error('Could not play the source video stream'));
+            });
         });
-        return () => {
-            renderer.dispose();
+        return {
+            output,
+            stop: () => {
+                renderer?.dispose();
+                videoRef.current && (videoRef.current.srcObject = null);
+                outputStream && disposeOfMediaStream(outputStream);
+            },
         };
-    }, [
-        backgroundBlurLevel,
-        backgroundFilter,
-        backgroundImageRef,
-        canvasRef,
-        tfLite,
-        videoRef,
-    ]);
-    return null;
+    }, [backgroundBlurLevel, backgroundFilter, backgroundImage, tfLite]);
+    const children = (jsxs("div", { className: "str-video__background-filters", children: [jsx("video", { className: clsx('str-video__background-filters__video', videoSize.height > videoSize.width &&
+                    'str-video__background-filters__video--tall'), ref: videoRef, playsInline: true, muted: true, controls: false, ...videoSize }), backgroundImage && (jsx("img", { className: "str-video__background-filters__background-image", alt: "Background", ref: bgImageRef, src: backgroundImage, ...videoSize })), jsx("canvas", { className: "str-video__background-filters__target-canvas", ...videoSize, ref: canvasRef })] }));
+    return {
+        start,
+        children,
+    };
 };
 
 const useFloatingUIPreset = ({ middleware = [], placement, strategy, offset: offsetInPx = 10, }) => {
@@ -2622,7 +2612,7 @@ const LivestreamPlayer = (props) => {
     return (jsx(StreamCall, { call: call, children: jsx(LivestreamLayout, { ...layoutProps }) }));
 };
 
-const [major, minor, patch] = ("1.2.12" ).split('.');
+const [major, minor, patch] = ("1.2.13" ).split('.');
 setSdkInfo({
     type: SfuModels.SdkType.REACT,
     major,