@stream-io/video-filters-web 0.3.0 → 0.5.0

package/src/compatibility.ts

@@ -46,3 +46,25 @@ export const isPlatformSupported = async ({
   !!window.WebGL2RenderingContext && // WebGL2 is required for the video filters
   !!document.createElement('canvas').getContext('webgl2') &&
   (await simd()); // SIMD is required for the wasm module
+
+/**
+ * Runs a check to see if the current platform supports
+ * the necessary APIs required for the MediaPipe-based video filters.
+ */
+export const isMediaPipePlatformSupported = async ({
+  forceMobileSupport = false,
+  forceSafariSupport = false,
+}: PlatformSupportFlags = {}) =>
+  typeof document !== 'undefined' &&
+  typeof window !== 'undefined' &&
+  typeof navigator !== 'undefined' &&
+  // we don't support mobile devices yet due to performance issues
+  (forceMobileSupport || !isMobile()) &&
+  // Safari has issues with timer throttling, causing low FPS when the tab goes to the background
+  (forceSafariSupport || !isSafari()) &&
+  typeof WebAssembly !== 'undefined' &&
+  typeof OffscreenCanvas !== 'undefined' && // OffscreenCanvas is required for efficient rendering
+  !!window.WebGL2RenderingContext && // WebGL2 is required for the video filters
+  !!new OffscreenCanvas(1, 1).getContext('webgl2') &&
+  typeof VideoFrame !== 'undefined' && // VideoFrame API is required for frame processing
+  typeof createImageBitmap !== 'undefined'; // createImageBitmap is required for background image processing

package/src/{createRenderer.ts → legacy/createRenderer.ts}

@@ -2,9 +2,8 @@ import { WorkerTimer } from '@stream-io/worker-timer';
 import { TFLite } from './tflite';
 import { buildWebGL2Pipeline } from './webgl2/webgl2Pipeline';
 import { getSegmentationParams, SegmentationLevel } from './segmentation';
+import { BackgroundBlurLevel, BackgroundFilter } from '../types';
 
-export type BackgroundFilter = 'blur' | 'image';
-export type BackgroundBlurLevel = 'low' | 'medium' | 'high' | number;
 export type Renderer = {
   /**
    * Disposes of the renderer.

package/src/{tflite.ts → legacy/tflite.ts}

@@ -1,6 +1,6 @@
 // @ts-expect-error - module is not declared
 import { createTFLiteSIMDModule } from './tflite-simd.js';
-import { packageName, version } from './version';
+import { packageName, version } from '../version';
 
 // This is a WebAssembly module compiled from the TensorFlow Lite C++ library.
 const createTFLite = createTFLiteSIMDModule as (opts?: any) => Promise<TFLite>;

package/src/{webgl2 → legacy/webgl2}/backgroundBlurStage.ts

@@ -4,7 +4,7 @@ import {
   createTexture,
   glsl,
 } from '../helpers/webglHelper';
-import type { BackgroundBlurLevel } from '../createRenderer';
+import { BackgroundBlurLevel } from '../../types';
 
 export type BackgroundBlurStage = {
   render(): void;

package/src/{webgl2 → legacy/webgl2}/webgl2Pipeline.ts

@@ -11,8 +11,8 @@ import {
 import { buildJointBilateralFilterStage } from './jointBilateralFilterStage';
 import { buildResizingStage } from './resizingStage';
 import { buildSoftmaxStage } from './softmaxStage';
-import { BackgroundBlurLevel, BackgroundFilter } from '../createRenderer';
 import { SegmentationParams } from '../segmentation';
+import type { BackgroundBlurLevel, BackgroundFilter } from '../../types';
 
 export function buildWebGL2Pipeline(
   videoSource: HTMLVideoElement,

package/src/mediapipe.ts

@@ -0,0 +1,25 @@
+import { packageName, version } from './version';
+
+let lastModelFilePath = '';
+let modelFileCache: ArrayBuffer | undefined;
+export const loadMediaPipe = async (
+  options: {
+    basePath?: string;
+    modelPath?: string;
+  } = {},
+): Promise<ArrayBuffer> => {
+  const {
+    basePath = `https://unpkg.com/${packageName}@${version}/mediapipe`,
+    modelPath = `${basePath}/models/selfie_segmenter.tflite`,
+  } = options;
+
+  const model =
+    modelPath === lastModelFilePath && modelFileCache
+      ? modelFileCache
+      : await fetch(modelPath).then((r) => r.arrayBuffer());
+
+  modelFileCache = model;
+  lastModelFilePath = modelPath;
+
+  return model;
+};

package/src/types.ts

@@ -0,0 +1,62 @@
+export type BackgroundSource = {
+  type: string;
+  media?: ImageBitmap | ReadableStream;
+  url: string;
+  video?: HTMLVideoElement;
+  track?: MediaStreamTrack;
+};
+export type BackgroundFilter = 'blur' | 'image';
+export type BackgroundBlurLevel = 'low' | 'medium' | 'high' | number;
+
+export interface SegmenterOptions {
+  backgroundSource?: BackgroundSource | null;
+  bgBlur: number;
+  bgBlurRadius: number;
+  isSelfieMode: boolean;
+}
+/**
+ * Static configuration for the processor, defining which background
+ * effect should be applied and how it should behave.
+ */
+export interface BackgroundOptions {
+  basePath?: string;
+  modelPath?: string;
+  backgroundFilter?: BackgroundFilter;
+  backgroundBlurLevel?: BackgroundBlurLevel;
+  backgroundImage?: string | undefined;
+}
+
+/**
+ * Performance statistics for video processing.
+ */
+export interface PerformanceStats {
+  delay: number;
+  fps: number;
+  timestamp: number;
+}
+
+/**
+ * Runtime hooks for handling lifecycle or error events.
+ */
+export interface VideoTrackProcessorHooks {
+  onError?: (error: unknown) => void;
+  onStats?: (stats: PerformanceStats) => void;
+}
+
+export const BACKGROUND_BLUR_MAP: Record<
+  BackgroundBlurLevel,
+  { bgBlur: number; bgBlurRadius: number }
+> = {
+  low: {
+    bgBlur: 15,
+    bgBlurRadius: 5,
+  },
+  medium: {
+    bgBlur: 20,
+    bgBlurRadius: 7,
+  },
+  high: {
+    bgBlur: 25,
+    bgBlurRadius: 10,
+  },
+};