@stream-io/video-react-sdk 0.5.8 → 0.5.10

package/dist/src/components/BackgroundFilters/BackgroundFilters.d.ts

@@ -0,0 +1,86 @@
+import { PropsWithChildren } from 'react';
+import { BackgroundBlurLevel, BackgroundFilter } from '@stream-io/video-filters-web';
+export type BackgroundFiltersProps = {
+    /**
+     * Enables or disables the background-blurring feature.
+     * @default true.
+     */
+    isBlurringEnabled?: boolean;
+    /**
+     * A list of URLs to use as background images.
+     */
+    backgroundImages?: string[];
+    /**
+     * The background filter to apply to the video (by default).
+     * @default 'none'.
+     */
+    backgroundFilter?: BackgroundFilter;
+    /**
+     * The URL of the image to use as the background (by default).
+     */
+    backgroundImage?: string;
+    /**
+     * The level of blur to apply to the background (by default).
+     * @default 'high'.
+     */
+    backgroundBlurLevel?: BackgroundBlurLevel;
+    /**
+     * The base path for the TensorFlow Lite files.
+     * @default 'https://unpkg.com/@stream-io/video-filters-web/tf'.
+     */
+    basePath?: string;
+    /**
+     * The path to the TensorFlow Lite WebAssembly file.
+     *
+     * Override this prop to use a custom path to the TensorFlow Lite WebAssembly file
+     * (e.g., if you choose to host it yourself).
+     */
+    tfFilePath?: string;
+    /**
+     * The path to the TensorFlow Lite model file.
+     * Override this prop to use a custom path to the TensorFlow Lite model file
+     * (e.g., if you choose to host it yourself).
+     */
+    modelFilePath?: string;
+};
+export type BackgroundFiltersAPI = {
+    /**
+     * Whether the current platform supports the background filters.
+     */
+    isSupported: boolean;
+    /**
+     * Indicates whether the background filters engine is loaded and ready.
+     */
+    isReady: boolean;
+    /**
+     * Disables all background filters applied to the video.
+     */
+    disableBackgroundFilter: () => void;
+    /**
+     * Applies a background blur filter to the video.
+     *
+     * @param blurLevel the level of blur to apply to the background.
+     */
+    applyBackgroundBlurFilter: (blurLevel: BackgroundBlurLevel) => void;
+    /**
+     * Applies a background image filter to the video.
+     *
+     * @param imageUrl the URL of the image to use as the background.
+     */
+    applyBackgroundImageFilter: (imageUrl: string) => void;
+};
+/**
+ * The context value for the background filters context.
+ */
+export type BackgroundFiltersContextValue = BackgroundFiltersProps & BackgroundFiltersAPI;
+/**
+ * A hook to access the background filters context API.
+ */
+export declare const useBackgroundFilters: () => BackgroundFiltersContextValue;
+/**
+ * A provider component that enables the use of background filters in your app.
+ *
+ * Please make sure you have the `@stream-io/video-filters-web` package installed
+ * in your project before using this component.
+ */
+export declare const BackgroundFiltersProvider: (props: PropsWithChildren<BackgroundFiltersProps>) => import("react/jsx-runtime").JSX.Element;

package/dist/src/components/BackgroundFilters/index.d.ts

@@ -0,0 +1 @@
+export * from './BackgroundFilters';

package/dist/src/components/Tooltip/hooks/useEnterLeaveHandlers.d.ts

@@ -1,5 +1,5 @@
 import { MouseEventHandler } from 'react';
-export declare const useEnterLeaveHandlers: <T extends HTMLElement>({ onMouseEnter, onMouseLeave, }?: Partial<Record<"onMouseEnter" | "onMouseLeave", MouseEventHandler<T>>>) => {
+export declare const useEnterLeaveHandlers: <T extends HTMLElement>({ onMouseEnter, onMouseLeave, }?: Partial<Record<'onMouseEnter' | 'onMouseLeave', MouseEventHandler<T>>>) => {
     handleMouseEnter: MouseEventHandler<T>;
     handleMouseLeave: MouseEventHandler<T>;
     tooltipVisible: boolean;

package/dist/src/components/index.d.ts

@@ -1,4 +1,5 @@
 export * from './Avatar';
+export * from './BackgroundFilters';
 export * from './Button';
 export * from './CallControls';
 export * from './CallParticipantsList';

package/dist/src/utilities/isComponentType.d.ts

@@ -1,2 +1,2 @@
 import { ComponentType, ReactElement } from 'react';
-export declare const isComponentType: <T extends {}>(elementOrComponent?: ReactElement<any, string | import("react").JSXElementConstructor<any>> | ComponentType<T> | null | undefined) => elementOrComponent is ComponentType<T>;
+export declare const isComponentType: <T extends {}>(elementOrComponent?: ComponentType<T> | ReactElement | null) => elementOrComponent is ComponentType<T>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-react-sdk",
-  "version": "0.5.8",
+  "version": "0.5.10",
   "packageManager": "yarn@3.2.4",
   "main": "./dist/index.cjs.js",
   "module": "./dist/index.es.js",
@@ -9,7 +9,7 @@
   "scripts": {
     "clean": "rimraf dist",
     "start": "rollup -c -w",
-    "build": "NODE_ENV=production rollup -c && yarn copy-css",
+    "build": "rimraf dist && NODE_ENV=production rollup -c && yarn copy-css",
     "copy-css": "cp -R ../../node_modules/@stream-io/video-styling/dist/* dist/",
     "start:docs": "npx stream-chat-docusaurus -s"
   },
@@ -29,8 +29,9 @@
   ],
   "dependencies": {
     "@floating-ui/react": "^0.26.5",
-    "@stream-io/video-client": "^0.6.6",
-    "@stream-io/video-react-bindings": "^0.4.6",
+    "@stream-io/video-client": "0.6.8",
+    "@stream-io/video-filters-web": "0.0.1",
+    "@stream-io/video-react-bindings": "0.4.8",
     "chart.js": "^4.4.1",
     "clsx": "^2.0.0",
     "react-chartjs-2": "^5.2.0"
@@ -40,14 +41,14 @@
     "react-dom": "^18.0.0"
   },
   "devDependencies": {
-    "@rollup/plugin-json": "^6.0.1",
-    "@rollup/plugin-replace": "^5.0.4",
-    "@rollup/plugin-typescript": "^11.1.5",
-    "@stream-io/video-styling": "^1.0.1",
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-replace": "^5.0.5",
+    "@rollup/plugin-typescript": "^11.1.6",
+    "@stream-io/video-styling": "^1.0.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "rimraf": "^5.0.5",
     "rollup": "^3.29.4",
-    "typescript": "^5.2.2"
+    "typescript": "^5.4.3"
   }
 }

package/src/components/BackgroundFilters/BackgroundFilters.tsx

@@ -0,0 +1,354 @@
+import {
+  createContext,
+  PropsWithChildren,
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+} from 'react';
+import clsx from 'clsx';
+import { useCall } from '@stream-io/video-react-bindings';
+import {
+  BackgroundBlurLevel,
+  BackgroundFilter,
+  createRenderer,
+  isPlatformSupported,
+  loadTFLite,
+  TFLite,
+} from '@stream-io/video-filters-web';
+
+export type BackgroundFiltersProps = {
+  /**
+   * Enables or disables the background-blurring feature.
+   * @default true.
+   */
+  isBlurringEnabled?: boolean;
+
+  /**
+   * A list of URLs to use as background images.
+   */
+  backgroundImages?: string[];
+
+  /**
+   * The background filter to apply to the video (by default).
+   * @default 'none'.
+   */
+  backgroundFilter?: BackgroundFilter;
+
+  /**
+   * The URL of the image to use as the background (by default).
+   */
+  backgroundImage?: string;
+
+  /**
+   * The level of blur to apply to the background (by default).
+   * @default 'high'.
+   */
+  backgroundBlurLevel?: BackgroundBlurLevel;
+
+  /**
+   * The base path for the TensorFlow Lite files.
+   * @default 'https://unpkg.com/@stream-io/video-filters-web/tf'.
+   */
+  basePath?: string;
+
+  /**
+   * The path to the TensorFlow Lite WebAssembly file.
+   *
+   * Override this prop to use a custom path to the TensorFlow Lite WebAssembly file
+   * (e.g., if you choose to host it yourself).
+   */
+  tfFilePath?: string;
+
+  /**
+   * The path to the TensorFlow Lite model file.
+   * Override this prop to use a custom path to the TensorFlow Lite model file
+   * (e.g., if you choose to host it yourself).
+   */
+  modelFilePath?: string;
+};
+
+export type BackgroundFiltersAPI = {
+  /**
+   * Whether the current platform supports the background filters.
+   */
+  isSupported: boolean;
+
+  /**
+   * Indicates whether the background filters engine is loaded and ready.
+   */
+  isReady: boolean;
+
+  /**
+   * Disables all background filters applied to the video.
+   */
+  disableBackgroundFilter: () => void;
+
+  /**
+   * Applies a background blur filter to the video.
+   *
+   * @param blurLevel the level of blur to apply to the background.
+   */
+  applyBackgroundBlurFilter: (blurLevel: BackgroundBlurLevel) => void;
+
+  /**
+   * Applies a background image filter to the video.
+   *
+   * @param imageUrl the URL of the image to use as the background.
+   */
+  applyBackgroundImageFilter: (imageUrl: string) => void;
+};
+
+/**
+ * The context value for the background filters context.
+ */
+export type BackgroundFiltersContextValue = BackgroundFiltersProps &
+  BackgroundFiltersAPI;
+
+/**
+ * The context for the background filters.
+ */
+const BackgroundFiltersContext = createContext<
+  BackgroundFiltersContextValue | undefined
+>(undefined);
+
+/**
+ * A hook to access the background filters context API.
+ */
+export const useBackgroundFilters = () => {
+  const context = useContext(BackgroundFiltersContext);
+  if (!context) {
+    throw new Error(
+      'useBackgroundFilters must be used within a BackgroundFiltersProvider',
+    );
+  }
+  return context;
+};
+
+/**
+ * A provider component that enables the use of background filters in your app.
+ *
+ * Please make sure you have the `@stream-io/video-filters-web` package installed
+ * in your project before using this component.
+ */
+export const BackgroundFiltersProvider = (
+  props: PropsWithChildren<BackgroundFiltersProps>,
+) => {
+  const {
+    children,
+    isBlurringEnabled = true,
+    backgroundImages = [],
+    backgroundFilter: bgFilterFromProps = undefined,
+    backgroundImage: bgImageFromProps = undefined,
+    backgroundBlurLevel: bgBlurLevelFromProps = 'high',
+    tfFilePath,
+    modelFilePath,
+    basePath,
+  } = props;
+
+  const [backgroundFilter, setBackgroundFilter] = useState(bgFilterFromProps);
+  const [backgroundImage, setBackgroundImage] = useState(bgImageFromProps);
+  const [backgroundBlurLevel, setBackgroundBlurLevel] =
+    useState(bgBlurLevelFromProps);
+
+  const applyBackgroundImageFilter = useCallback((imageUrl: string) => {
+    setBackgroundFilter('image');
+    setBackgroundImage(imageUrl);
+  }, []);
+
+  const applyBackgroundBlurFilter = useCallback(
+    (blurLevel: BackgroundBlurLevel = 'high') => {
+      setBackgroundFilter('blur');
+      setBackgroundBlurLevel(blurLevel);
+    },
+    [],
+  );
+
+  const disableBackgroundFilter = useCallback(() => {
+    setBackgroundFilter(undefined);
+    setBackgroundImage(undefined);
+    setBackgroundBlurLevel('high');
+  }, []);
+
+  const [isSupported, setIsSupported] = useState(false);
+  useEffect(() => {
+    isPlatformSupported().then(setIsSupported);
+  }, []);
+
+  const [tfLite, setTfLite] = useState<TFLite>();
+  useEffect(() => {
+    // don't try to load TFLite if the platform is not supported
+    if (!isSupported) return;
+    loadTFLite({ basePath, modelFilePath, tfFilePath })
+      .then(setTfLite)
+      .catch((err) => console.error('Failed to load TFLite', err));
+  }, [basePath, isSupported, modelFilePath, tfFilePath]);
+
+  return (
+    <BackgroundFiltersContext.Provider
+      value={{
+        isSupported,
+        isReady: !!tfLite,
+        backgroundImage,
+        backgroundBlurLevel,
+        backgroundFilter,
+        disableBackgroundFilter,
+        applyBackgroundBlurFilter,
+        applyBackgroundImageFilter,
+        backgroundImages,
+        isBlurringEnabled,
+        tfFilePath,
+        modelFilePath,
+        basePath,
+      }}
+    >
+      {children}
+      {tfLite && backgroundFilter && <BackgroundFilters tfLite={tfLite} />}
+    </BackgroundFiltersContext.Provider>
+  );
+};
+
+const BackgroundFilters = (props: { tfLite: TFLite }) => {
+  const { tfLite } = props;
+  const call = useCall();
+  const { backgroundImage, backgroundFilter } = useBackgroundFilters();
+  const [videoRef, setVideoRef] = useState<HTMLVideoElement | null>(null);
+  const [bgImageRef, setBgImageRef] = useState<HTMLImageElement | null>(null);
+  const [canvasRef, setCanvasRef] = useState<HTMLCanvasElement | null>(null);
+  const [width, setWidth] = useState(1920);
+  const [height, setHeight] = useState(1080);
+
+  const resolveFilterRef =
+    useRef<(value: MediaStream | PromiseLike<MediaStream>) => void>();
+
+  const [mediaStream, setMediaStream] = useState<MediaStream>();
+  const registerFilterRef = useRef(Promise.resolve(async () => {}));
+  useEffect(() => {
+    if (!call || !backgroundFilter) return;
+    registerFilterRef.current = registerFilterRef.current.then(() =>
+      call.camera.registerFilter(async (ms) => {
+        return new Promise<MediaStream>((resolve) => {
+          setMediaStream(ms);
+          resolveFilterRef.current = resolve;
+        });
+      }),
+    );
+
+    return () => {
+      registerFilterRef.current
+        .then((unregister) => unregister())
+        .then(() => setMediaStream(undefined))
+        .catch((err) => console.error('Failed to unregister filter', err));
+    };
+  }, [backgroundFilter, call]);
+
+  useEffect(() => {
+    if (!mediaStream || !videoRef || !canvasRef) return;
+
+    const handleOnPlay = () => {
+      const [track] = mediaStream.getVideoTracks();
+      if (track) {
+        const { width: w = 0, height: h = 0 } = track.getSettings();
+        setWidth(w);
+        setHeight(h);
+      }
+
+      const resolveFilter = resolveFilterRef.current;
+      if (!resolveFilter) return;
+      const filter = canvasRef.captureStream();
+      resolveFilter(filter);
+    };
+    videoRef.addEventListener('play', handleOnPlay);
+
+    videoRef.srcObject = mediaStream;
+    videoRef.play().catch((err) => console.error('Failed to play video', err));
+    return () => {
+      videoRef.removeEventListener('play', handleOnPlay);
+      videoRef.srcObject = null;
+    };
+  }, [canvasRef, mediaStream, videoRef]);
+
+  return (
+    <div
+      className="str-video__background-filters"
+      style={{
+        width: `${width}px`,
+        height: `${height}px`,
+      }}
+    >
+      {mediaStream && (
+        <RenderPipeline
+          tfLite={tfLite}
+          videoRef={videoRef}
+          canvasRef={canvasRef}
+          backgroundImageRef={bgImageRef}
+        />
+      )}
+      <video
+        className={clsx(
+          'str-video__background-filters__video',
+          height > width && 'str-video__background-filters__video--tall',
+        )}
+        ref={setVideoRef}
+        autoPlay
+        playsInline
+        controls={false}
+        width={width}
+        height={height}
+        muted
+        loop
+      />
+      {backgroundImage && (
+        <img
+          className="str-video__background-filters__background-image"
+          key={backgroundImage}
+          alt="Background"
+          ref={setBgImageRef}
+          src={backgroundImage}
+          width={width}
+          height={height}
+        />
+      )}
+      <canvas
+        className="str-video__background-filters__target-canvas"
+        key={`key-${width}${height}`}
+        width={width}
+        height={height}
+        ref={setCanvasRef}
+      />
+    </div>
+  );
+};
+
+const RenderPipeline = (props: {
+  tfLite: TFLite;
+  videoRef: HTMLVideoElement | null;
+  canvasRef: HTMLCanvasElement | null;
+  backgroundImageRef: HTMLImageElement | null;
+}) => {
+  const { tfLite, videoRef, canvasRef, backgroundImageRef } = props;
+  const { backgroundFilter, backgroundBlurLevel } = useBackgroundFilters();
+  useEffect(() => {
+    if (!videoRef || !canvasRef || !backgroundFilter) return;
+    if (backgroundFilter === 'image' && !backgroundImageRef) return;
+
+    const renderer = createRenderer(tfLite, videoRef, canvasRef, {
+      backgroundFilter,
+      backgroundImage: backgroundImageRef ?? undefined,
+      backgroundBlurLevel,
+    });
+    return () => {
+      renderer.dispose();
+    };
+  }, [
+    backgroundBlurLevel,
+    backgroundFilter,
+    backgroundImageRef,
+    canvasRef,
+    tfLite,
+    videoRef,
+  ]);
+
+  return null;
+};

package/src/components/BackgroundFilters/index.ts

@@ -0,0 +1 @@
+export * from './BackgroundFilters';

package/src/components/index.ts

@@ -1,4 +1,5 @@
 export * from './Avatar';
+export * from './BackgroundFilters';
 export * from './Button';
 export * from './CallControls';
 export * from './CallParticipantsList';