@stream-io/video-react-sdk 1.41.0 → 1.43.0

package/dist/index.d.ts

@@ -4,5 +4,5 @@ export * from './src/core';
 export * from './src/components';
 export * from './src/wrappers';
 export * from './src/translations';
-export { useHorizontalScrollPosition, useVerticalScrollPosition, useRequestPermission, usePersistedDevicePreferences, useDeviceList, useModeration, } from './src/hooks';
+export { useHorizontalScrollPosition, useVerticalScrollPosition, useRequestPermission, usePersistedDevicePreferences, useDeviceList, useModeration, useLoopbackRecording, type LoopbackRecordingState, type StartLoopbackRecordingOptions, type UseLoopbackRecordingResult, } from './src/hooks';
 export { applyFilter, type Filter } from './src/utilities/filter';

package/dist/src/hooks/index.d.ts

@@ -5,3 +5,4 @@ export * from './useRequestPermission';
 export * from './useDeviceList';
 export * from './useModeration';
 export * from './useDragToScroll';
+export * from './useLoopbackRecording';

package/dist/src/hooks/useLoopbackRecording.d.ts

@@ -0,0 +1,23 @@
+import { type MediaRecordingResult } from '@stream-io/video-client';
+/** Lifecycle phase of a loopback recording. */
+export type LoopbackRecordingState = 'idle' | 'awaiting-streams' | 'recording';
+export type StartLoopbackRecordingOptions = {
+    /** Whether to include loopback video. Audio is always recorded. */
+    includeVideo?: boolean;
+    /** Maximum recording duration in milliseconds. */
+    maxDurationMs?: number;
+};
+export type UseLoopbackRecordingResult = {
+    /** Starts recording and resolves with the recorded blob. */
+    startRecording: (options?: StartLoopbackRecordingOptions) => Promise<MediaRecordingResult | null>;
+    /** Stops the current recording or stream wait. */
+    stopRecording: () => Promise<void>;
+    /** Lifecycle phase of the recording. */
+    recordingState: LoopbackRecordingState;
+    /** SFU-echoed video stream on the local participant. */
+    loopbackVideoStream?: MediaStream;
+    /** SFU-echoed audio stream on the local participant. */
+    loopbackAudioStream?: MediaStream;
+};
+/** Records SFU loopback streams for pre-call testing. */
+export declare const useLoopbackRecording: () => UseLoopbackRecordingResult;

package/index.ts

@@ -15,6 +15,10 @@ export {
   usePersistedDevicePreferences,
   useDeviceList,
   useModeration,
+  useLoopbackRecording,
+  type LoopbackRecordingState,
+  type StartLoopbackRecordingOptions,
+  type UseLoopbackRecordingResult,
 } from './src/hooks';
 export { applyFilter, type Filter } from './src/utilities/filter';
 

package/package.json

@@ -1,18 +1,18 @@
 {
   "name": "@stream-io/video-react-sdk",
-  "version": "1.41.0",
+  "version": "1.43.0",
   "main": "./dist/index.cjs.js",
-  "module": "./dist/index.es.js",
+  "module": "./dist/esm/index.js",
   "types": "./dist/index.d.ts",
   "exports": {
     ".": {
-      "import": "./dist/index.es.js",
-      "require": "./dist/index.cjs.js",
-      "types": "./dist/index.d.ts"
+      "types": "./dist/index.d.ts",
+      "import": "./dist/esm/index.js",
+      "require": "./dist/index.cjs.js"
     },
     "./embedded": {
       "types": "./dist/embedded.d.ts",
-      "import": "./dist/embedded.es.js",
+      "import": "./dist/esm/embedded.js",
       "require": "./dist/embedded.cjs.js"
     },
     "./dist/css/*": "./dist/css/*"
@@ -41,13 +41,17 @@
     "CHANGELOG.md"
   ],
   "sideEffects": [
-    "*.css"
+    "*.css",
+    "./dist/esm/index.js",
+    "./dist/esm/embedded.js",
+    "./dist/index.cjs.js",
+    "./dist/embedded.cjs.js"
   ],
   "dependencies": {
     "@floating-ui/react": "^0.27.20",
-    "@stream-io/video-client": "1.58.0",
-    "@stream-io/video-filters-web": "0.8.6",
-    "@stream-io/video-react-bindings": "1.20.0",
+    "@stream-io/video-client": "1.59.1",
+    "@stream-io/video-filters-web": "0.10.0",
+    "@stream-io/video-react-bindings": "1.20.2",
     "chart.js": "^4.5.1",
     "clsx": "^2.1.1",
     "react-chartjs-2": "^5.3.1"

package/src/components/BackgroundFilters/BackgroundFilters.tsx

@@ -12,6 +12,7 @@ import { useCall, useCallStateHooks } from '@stream-io/video-react-bindings';
 import { Call, disposeOfMediaStream } from '@stream-io/video-client';
 import {
   BackgroundBlurLevel,
+  BackgroundEffectOptions,
   createRenderer,
   isMediaPipePlatformSupported,
   isPlatformSupported,
@@ -431,8 +432,10 @@ const BackgroundFilters = (props: {
   >(undefined);
   handleStatsRef.current = onStats;
 
+  const filterActive = !!backgroundFilter;
+
   useEffect(() => {
-    if (!call || !backgroundFilter) return;
+    if (!call || !filterActive) return;
 
     setIsLoading(true);
     const { unregister, registered } = call.camera.registerFilter((ms) => {
@@ -449,7 +452,7 @@ const BackgroundFilters = (props: {
     return () => {
       unregister().catch((err) => console.warn(`Can't unregister filter`, err));
     };
-  }, [call, start, backgroundFilter, setIsLoading]);
+  }, [call, start, filterActive, setIsLoading]);
 
   return children;
 };
@@ -469,6 +472,16 @@ const useRenderer = (
     segmentationOptions,
   } = api;
 
+  const optionsRef = useRef<BackgroundEffectOptions>({});
+  optionsRef.current = {
+    backgroundFilter,
+    backgroundBlurLevel,
+    backgroundImage,
+    segmentationOptions,
+  };
+
+  const processorRef = useRef<VirtualBackground | undefined>(undefined);
+
   const videoRef = useRef<HTMLVideoElement>(null);
   const canvasRef = useRef<HTMLCanvasElement>(null);
   const bgImageRef = useRef<HTMLImageElement>(null);
@@ -479,7 +492,7 @@ const useRenderer = (
     },
   );
 
-  const start = useCallback(
+  const startMediaPipe = useCallback(
     (
       ms: MediaStream,
       onError?: (error: any) => void,
@@ -487,6 +500,65 @@ const useRenderer = (
     ) => {
       let outputStream: MediaStream | undefined;
       let processor: VirtualBackground | undefined;
+
+      const output = new Promise<MediaStream>((resolve, reject) => {
+        const options = optionsRef.current;
+        if (!options.backgroundFilter) {
+          reject(new Error('No filter specified'));
+          return;
+        }
+
+        const [track] = ms.getVideoTracks();
+        if (!track) {
+          reject(new Error('No video tracks in input media stream'));
+          return;
+        }
+
+        call?.tracer.trace('backgroundFilters.enable', {
+          ...options,
+          engine: FilterEngine.MEDIA_PIPE,
+          modelFilePath,
+        });
+
+        const trackSettings = track.getSettings();
+        flushSync(() =>
+          setVideoSize({
+            width: trackSettings.width ?? 0,
+            height: trackSettings.height ?? 0,
+          }),
+        );
+
+        processor = new VirtualBackground(
+          track,
+          { ...options, basePath, modelPath: modelFilePath },
+          { onError, onStats },
+        );
+        processorRef.current = processor;
+        processor
+          .start()
+          .then((processedTrack) => {
+            outputStream = new MediaStream([processedTrack]);
+            resolve(outputStream);
+          })
+          .catch(reject);
+      });
+
+      return {
+        output,
+        stop: () => {
+          call?.tracer.trace('backgroundFilters.disable', null);
+          processorRef.current = undefined;
+          processor?.stop();
+          if (outputStream) disposeOfMediaStream(outputStream);
+        },
+      };
+    },
+    [call?.tracer, modelFilePath, basePath],
+  );
+
+  const startTf = useCallback(
+    (ms: MediaStream, onError?: (error: any) => void) => {
+      let outputStream: MediaStream | undefined;
       let renderer: Renderer | undefined;
 
       const output = new Promise<MediaStream>((resolve, reject) => {
@@ -500,117 +572,63 @@ const useRenderer = (
         const bgImageEl = bgImageRef.current;
 
         const [track] = ms.getVideoTracks();
-
         if (!track) {
           reject(new Error('No video tracks in input media stream'));
           return;
         }
 
-        if (engine === FilterEngine.MEDIA_PIPE) {
-          call?.tracer.trace('backgroundFilters.enable', {
-            backgroundFilter,
-            backgroundBlurLevel,
-            backgroundImage,
-            engine,
-            modelFilePath,
-          });
-
-          if (!videoEl) {
-            reject(new Error('Renderer started before elements are ready'));
-            return;
-          }
-
-          const trackSettings = track.getSettings();
-          flushSync(() =>
-            setVideoSize({
-              width: trackSettings.width ?? 0,
-              height: trackSettings.height ?? 0,
-            }),
-          );
-
-          processor = new VirtualBackground(
-            track,
-            {
-              basePath: basePath,
-              modelPath: modelFilePath,
+        if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
+          reject(new Error('Renderer started before elements are ready'));
+          return;
+        }
+
+        videoEl.srcObject = ms;
+        videoEl.play().then(
+          () => {
+            const trackSettings = track.getSettings();
+            flushSync(() =>
+              setVideoSize({
+                width: trackSettings.width ?? 0,
+                height: trackSettings.height ?? 0,
+              }),
+            );
+            call?.tracer.trace('backgroundFilters.enable', {
+              backgroundFilter,
               backgroundBlurLevel,
               backgroundImage,
-              backgroundFilter,
-              segmentationOptions,
-            },
-            { onError, onStats },
-          );
-          processor
-            .start()
-            .then((processedTrack) => {
-              outputStream = new MediaStream([processedTrack]);
-              resolve(outputStream);
-            })
-            .catch((error) => {
-              reject(error);
+              engine: FilterEngine.TF,
             });
 
-          return;
-        }
+            if (!tfLite) {
+              reject(new Error('TensorFlow Lite not loaded'));
+              return;
+            }
 
-        if (engine === FilterEngine.TF) {
-          if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
-            reject(new Error('Renderer started before elements are ready'));
-            return;
-          }
-
-          videoEl.srcObject = ms;
-          videoEl.play().then(
-            () => {
-              const trackSettings = track.getSettings();
-              flushSync(() =>
-                setVideoSize({
-                  width: trackSettings.width ?? 0,
-                  height: trackSettings.height ?? 0,
-                }),
-              );
-              call?.tracer.trace('backgroundFilters.enable', {
+            renderer = createRenderer(
+              tfLite,
+              videoEl,
+              canvasEl,
+              {
                 backgroundFilter,
                 backgroundBlurLevel,
-                backgroundImage,
-                engine,
-              });
-
-              if (!tfLite) {
-                reject(new Error('TensorFlow Lite not loaded'));
-                return;
-              }
-
-              renderer = createRenderer(
-                tfLite,
-                videoEl,
-                canvasEl,
-                {
-                  backgroundFilter,
-                  backgroundBlurLevel,
-                  backgroundImage: bgImageEl ?? undefined,
-                },
-                onError,
-              );
-              outputStream = canvasEl.captureStream();
-
-              resolve(outputStream);
-            },
-            () => {
-              reject(new Error('Could not play the source video stream'));
-            },
-          );
-          return;
-        }
-
-        reject(new Error('No supported engine available'));
+                backgroundImage: bgImageEl ?? undefined,
+              },
+              onError,
+            );
+            outputStream = canvasEl.captureStream();
+
+            resolve(outputStream);
+          },
+          () => {
+            reject(new Error('Could not play the source video stream'));
+          },
+        );
       });
 
       return {
         output,
         stop: () => {
           call?.tracer.trace('backgroundFilters.disable', null);
-          processor?.stop();
           renderer?.dispose();
           if (videoRef.current) videoRef.current.srcObject = null;
           if (outputStream) disposeOfMediaStream(outputStream);
@@ -618,18 +636,36 @@ const useRenderer = (
       };
     },
     [
-      backgroundBlurLevel,
-      backgroundFilter,
-      backgroundImage,
       call?.tracer,
       tfLite,
-      engine,
-      modelFilePath,
-      basePath,
-      segmentationOptions,
+      backgroundFilter,
+      backgroundBlurLevel,
+      backgroundImage,
     ],
   );
 
+  const start = engine === FilterEngine.TF ? startTf : startMediaPipe;
+
+  useEffect(() => {
+    if (!backgroundFilter) return;
+
+    processorRef.current
+      ?.updateOptions({
+        backgroundFilter,
+        backgroundBlurLevel,
+        backgroundImage,
+        segmentationOptions,
+      })
+      .catch((err) =>
+        console.warn(`[filters] Can't apply the background options`, err),
+      );
+  }, [
+    backgroundFilter,
+    backgroundBlurLevel,
+    backgroundImage,
+    segmentationOptions,
+  ]);
+
   const children = (
     <div className="str-video__background-filters">
       <video

package/src/hooks/index.ts

@@ -5,3 +5,4 @@ export * from './useRequestPermission';
 export * from './useDeviceList';
 export * from './useModeration';
 export * from './useDragToScroll';
+export * from './useLoopbackRecording';

package/src/hooks/useLoopbackRecording.ts

@@ -0,0 +1,207 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import {
+  CallingState,
+  DEFAULT_LOOPBACK_RECORDING_DURATION_MS,
+  LOOPBACK_STREAMS_WAIT_TIMEOUT_MS,
+  MediaStreamRecorder,
+  clampLoopbackRecordingDuration,
+  getLoopbackStreams,
+  type MediaRecordingResult,
+  videoLoggerSystem,
+  waitForLoopbackStreams,
+  enableLoopbackAudio,
+  type LoopbackStreams,
+} from '@stream-io/video-client';
+import { useCall, useCallStateHooks } from '@stream-io/video-react-bindings';
+
+/** Lifecycle phase of a loopback recording. */
+export type LoopbackRecordingState = 'idle' | 'awaiting-streams' | 'recording';
+
+const attachHiddenMediaSink = (stream: MediaStream): (() => void) => {
+  const logger = videoLoggerSystem.getLogger('useLoopbackRecording');
+  const element = document.createElement('video');
+  element.muted = true;
+  element.autoplay = true;
+  element.playsInline = true;
+  element.srcObject = stream;
+
+  element.play().catch((error) => {
+    logger.warn('sink element failed to play; recording may be empty', error);
+  });
+
+  return () => {
+    element.pause();
+    element.srcObject = null;
+  };
+};
+
+export type StartLoopbackRecordingOptions = {
+  /** Whether to include loopback video. Audio is always recorded. */
+  includeVideo?: boolean;
+  /** Maximum recording duration in milliseconds. */
+  maxDurationMs?: number;
+};
+
+export type UseLoopbackRecordingResult = {
+  /** Starts recording and resolves with the recorded blob. */
+  startRecording: (
+    options?: StartLoopbackRecordingOptions,
+  ) => Promise<MediaRecordingResult | null>;
+  /** Stops the current recording or stream wait. */
+  stopRecording: () => Promise<void>;
+  /** Lifecycle phase of the recording. */
+  recordingState: LoopbackRecordingState;
+  /** SFU-echoed video stream on the local participant. */
+  loopbackVideoStream?: MediaStream;
+  /** SFU-echoed audio stream on the local participant. */
+  loopbackAudioStream?: MediaStream;
+};
+
+/** Records SFU loopback streams for pre-call testing. */
+export const useLoopbackRecording = (): UseLoopbackRecordingResult => {
+  const call = useCall();
+  const { useCallCallingState, useParticipantCount, useLocalParticipant } =
+    useCallStateHooks();
+  const callingState = useCallCallingState();
+  const participantCount = useParticipantCount();
+  const localParticipant = useLocalParticipant();
+
+  const [recordingState, setRecordingState] =
+    useState<LoopbackRecordingState>('idle');
+
+  const recordingStateRef = useRef<LoopbackRecordingState>('idle');
+  const isMountedRef = useRef(true);
+  const awaitAbortRef = useRef<AbortController | null>(null);
+  const stopRef = useRef<(() => void) | null>(null);
+  const runRef = useRef<Promise<unknown> | null>(null);
+
+  const loopbackStreams = useMemo<LoopbackStreams>(
+    () => getLoopbackStreams(call, localParticipant),
+    [call, localParticipant],
+  );
+
+  const updateState = useCallback((next: LoopbackRecordingState) => {
+    recordingStateRef.current = next;
+    if (isMountedRef.current) {
+      setRecordingState(next);
+    }
+  }, []);
+
+  const stopRecording = useCallback(async (): Promise<void> => {
+    const current = recordingStateRef.current;
+    if (current === 'idle') return;
+
+    if (current === 'awaiting-streams') {
+      videoLoggerSystem
+        .getLogger('useLoopbackRecording')
+        .debug('aborting the awaiting-streams wait');
+      awaitAbortRef.current?.abort();
+      return;
+    }
+
+    stopRef.current?.();
+    await runRef.current;
+  }, []);
+
+  const startRecording = useCallback(
+    async ({
+      includeVideo = true,
+      maxDurationMs = DEFAULT_LOOPBACK_RECORDING_DURATION_MS,
+    }: StartLoopbackRecordingOptions = {}): Promise<MediaRecordingResult | null> => {
+      const logger = videoLoggerSystem.getLogger('useLoopbackRecording');
+      if (!call) return null;
+
+      if (recordingStateRef.current !== 'idle') {
+        logger.warn('a recording is already running');
+        return null;
+      }
+
+      if (call.state.participantCount > 1) {
+        logger.warn('cannot start recording with other participants present');
+        return null;
+      }
+
+      const run = async (): Promise<MediaRecordingResult | null> => {
+        awaitAbortRef.current = new AbortController();
+        updateState('awaiting-streams');
+
+        try {
+          const tracks = await waitForLoopbackStreams(call, {
+            includeVideo,
+            signal: awaitAbortRef.current.signal,
+            timeoutMs: LOOPBACK_STREAMS_WAIT_TIMEOUT_MS,
+          });
+
+          if (tracks === null) {
+            logger.debug('loopback stream wait was aborted');
+            return null;
+          }
+
+          updateState('recording');
+
+          const restoreAudio = enableLoopbackAudio(tracks);
+          const stream = new MediaStream(
+            [tracks.audioTrack, tracks.videoTrack].filter(
+              (track): track is MediaStreamTrack => !!track,
+            ),
+          );
+          const detachSink = attachHiddenMediaSink(stream);
+          const recorder = new MediaStreamRecorder(stream, {
+            maxDurationMs: clampLoopbackRecordingDuration(maxDurationMs),
+          });
+          stopRef.current = recorder.stop;
+          try {
+            return await recorder.start();
+          } finally {
+            detachSink();
+            restoreAudio();
+          }
+        } catch (err) {
+          logger.error('loopback recording failed', err);
+          throw err;
+        } finally {
+          awaitAbortRef.current = null;
+          stopRef.current = null;
+          updateState('idle');
+        }
+      };
+
+      const promise = run();
+      runRef.current = promise.catch(() => undefined);
+      return promise;
+    },
+    [call, updateState],
+  );
+
+  useEffect(() => {
+    if (
+      callingState === CallingState.LEFT ||
+      callingState === CallingState.IDLE
+    ) {
+      stopRecording().catch(() => {});
+    }
+  }, [callingState, stopRecording]);
+
+  useEffect(() => {
+    if (recordingState !== 'idle' && participantCount > 1) {
+      stopRecording().catch(() => {});
+    }
+  }, [participantCount, recordingState, stopRecording]);
+
+  useEffect(() => {
+    isMountedRef.current = true;
+    return () => {
+      isMountedRef.current = false;
+      awaitAbortRef.current?.abort('the component unmounted');
+      stopRef.current?.();
+    };
+  }, []);
+
+  return {
+    startRecording,
+    stopRecording,
+    recordingState,
+    loopbackVideoStream: loopbackStreams.loopbackVideoStream,
+    loopbackAudioStream: loopbackStreams.loopbackAudioStream,
+  };
+};