@stream-io/video-filters-web 0.7.1 → 0.7.3

package/dist/src/BaseVideoProcessor.d.ts

@@ -29,7 +29,7 @@ export declare abstract class BaseVideoProcessor {
     protected constructor(track: MediaStreamVideoTrack, hooks?: VideoTrackProcessorHooks);
     start(): Promise<MediaStreamTrack>;
     stop(): void;
-    private updateStats;
+    protected updateStats(delay: number): void;
     protected abstract initialize(): Promise<void>;
     protected abstract transform(frame: VideoFrame): Promise<VideoFrame>;
     protected onFlush(): void;

package/dist/src/VirtualBackground.d.ts

@@ -14,7 +14,6 @@ export declare class VirtualBackground extends BaseVideoProcessor {
     private latestCategoryMask;
     private latestConfidenceMask;
     private lastFrameTime;
-    private count;
     constructor(track: MediaStreamVideoTrack, options?: BackgroundOptions, hooks?: VideoTrackProcessorHooks);
     protected initialize(): Promise<void>;
     private initializeSegmenter;

package/dist/src/WebGLRenderer.d.ts

@@ -75,6 +75,11 @@ export declare class WebGLRenderer {
         bgBlur: number;
         bgBlurRadius: number;
         isSelfieMode: boolean;
+        segmentationOptions?: {
+            smoothingFactor?: number;
+            smoothstepMin?: number;
+            smoothstepMax?: number;
+        };
     }, categoryTexture?: WebGLTexture, confidenceTexture?: WebGLTexture): void;
     close(): void;
 }

package/dist/src/types.d.ts

@@ -7,11 +7,41 @@ export type BackgroundSource = {
 };
 export type BackgroundFilter = 'blur' | 'image';
 export type BackgroundBlurLevel = 'low' | 'medium' | 'high' | number;
+/**
+ * Options for controlling the segmentation mask smoothing.
+ * These values are passed to the WebGL shader that blends
+ * consecutive segmentation masks over time.
+ */
+export interface SegmentationOptions {
+    /**
+     * Controls how fast the mask adapts to new segmentation results.
+     * Higher values make the mask react faster but may cause flickering.
+     * Lower values produce smoother transitions but increase latency.
+     * Value should be between 0 and 1.
+     * @default 0.8
+     */
+    smoothingFactor?: number;
+    /**
+     * Lower edge of the smoothstep function applied to the confidence mask.
+     * Confidence values below this are mapped to 0 (background).
+     * Value should be between 0 and 1, and less than smoothstepMax.
+     * @default 0.6
+     */
+    smoothstepMin?: number;
+    /**
+     * Upper edge of the smoothstep function applied to the confidence mask.
+     * Confidence values above this are mapped to 1 (foreground).
+     * Value should be between 0 and 1, and greater than smoothstepMin.
+     * @default 0.9
+     */
+    smoothstepMax?: number;
+}
 export interface SegmenterOptions {
     backgroundSource?: BackgroundSource | null;
     bgBlur: number;
     bgBlurRadius: number;
     isSelfieMode: boolean;
+    segmentationOptions?: SegmentationOptions;
 }
 /**
  * Static configuration for the processor, defining which background
@@ -23,6 +53,7 @@ export interface BackgroundOptions {
     backgroundFilter?: BackgroundFilter;
     backgroundBlurLevel?: BackgroundBlurLevel;
     backgroundImage?: string | undefined;
+    segmentationOptions?: SegmentationOptions;
 }
 /**
  * Performance statistics for video processing.
@@ -39,7 +70,7 @@ export interface VideoTrackProcessorHooks {
     onError?: (error: unknown) => void;
     onStats?: (stats: PerformanceStats) => void;
 }
-export declare const BACKGROUND_BLUR_MAP: Record<BackgroundBlurLevel, {
-    bgBlur: number;
-    bgBlurRadius: number;
-}>;
+/**
+ * Maps blur level to blur strength values.
+ */
+export declare const BACKGROUND_BLUR_MAP: Record<'low' | 'medium' | 'high', number>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-filters-web",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "main": "./dist/index.cjs.js",
   "module": "./dist/index.es.js",
   "types": "./dist/index.d.ts",

package/src/BaseVideoProcessor.ts

@@ -64,11 +64,8 @@ export abstract class BaseVideoProcessor {
             this.canvas.height = frame.displayHeight;
           }
 
-          const start = performance.now();
           const processed = await this.transform(frame);
-          const delay = performance.now() - start;
 
-          this.updateStats(delay);
           controller.enqueue(processed);
         } catch (e) {
           this.hooks.onError?.(e);
@@ -98,7 +95,7 @@ export abstract class BaseVideoProcessor {
     this.onStop();
   }
 
-  private updateStats(delay: number): void {
+  protected updateStats(delay: number): void {
     this.frames++;
     this.delayTotal += delay;
 

package/src/VirtualBackground.ts

@@ -24,7 +24,6 @@ export class VirtualBackground extends BaseVideoProcessor {
   private latestCategoryMask: WebGLTexture | undefined = undefined;
   private latestConfidenceMask: WebGLTexture | undefined = undefined;
   private lastFrameTime = -1;
-  private count = 0;
 
   constructor(
     track: MediaStreamVideoTrack,
@@ -71,10 +70,12 @@ export class VirtualBackground extends BaseVideoProcessor {
 
   protected async transform(frame: VideoFrame): Promise<VideoFrame> {
     const currentTime = frame.timestamp;
-    const hasNewFrame = currentTime !== this.lastFrameTime;
+    const hasNewFrame = currentTime - this.lastFrameTime > 1_000;
     this.lastFrameTime = currentTime;
 
     if (hasNewFrame && this.isSegmenterReady && this.segmenter) {
+      const start = performance.now();
+
       await this.runSegmentation(frame);
 
       this.webGlRenderer.render(
@@ -83,6 +84,8 @@ export class VirtualBackground extends BaseVideoProcessor {
         this.latestCategoryMask,
         this.latestConfidenceMask,
       );
+
+      this.updateStats(performance.now() - start);
     }
 
     return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
@@ -121,25 +124,22 @@ export class VirtualBackground extends BaseVideoProcessor {
         bgBlur: 0,
         bgBlurRadius: 0,
         isSelfieMode,
+        segmentationOptions: this.options.segmentationOptions,
       };
     }
 
     const blurLevel = this.options.backgroundBlurLevel;
+    const strength =
+      typeof blurLevel === 'string'
+        ? BACKGROUND_BLUR_MAP[blurLevel]
+        : Math.round(blurLevel ?? 5);
 
-    if (typeof blurLevel === 'string') {
-      return {
-        ...BACKGROUND_BLUR_MAP[blurLevel],
-        backgroundSource: undefined,
-        isSelfieMode,
-      };
-    }
-
-    const numeric = blurLevel ?? 5;
     return {
       backgroundSource: undefined,
-      bgBlur: Math.min(numeric * 3, 30),
-      bgBlurRadius: Math.min(numeric, 10),
+      bgBlur: Math.min(strength * 1.5, 20),
+      bgBlurRadius: Math.min(strength, 10),
       isSelfieMode,
+      segmentationOptions: this.options.segmentationOptions,
     };
   }
 

package/src/WebGLRenderer.ts

@@ -754,6 +754,11 @@ export class WebGLRenderer {
       bgBlur: number;
       bgBlurRadius: number;
       isSelfieMode: boolean;
+      segmentationOptions?: {
+        smoothingFactor?: number;
+        smoothstepMin?: number;
+        smoothstepMax?: number;
+      };
     },
     categoryTexture?: WebGLTexture,
     confidenceTexture?: WebGLTexture,
@@ -864,9 +869,18 @@ export class WebGLRenderer {
     gl.bindTexture(gl.TEXTURE_2D, prevStateTexture);
     gl.uniform1i(stateUpdateLocations.prevStateTexture, 2);
 
-    gl.uniform1f(stateUpdateLocations.smoothingFactor, 0.8);
-    gl.uniform1f(stateUpdateLocations.smoothstepMin, 0.6);
-    gl.uniform1f(stateUpdateLocations.smoothstepMax, 0.9);
+    gl.uniform1f(
+      stateUpdateLocations.smoothingFactor,
+      options.segmentationOptions?.smoothingFactor ?? 0.8,
+    );
+    gl.uniform1f(
+      stateUpdateLocations.smoothstepMin,
+      options.segmentationOptions?.smoothstepMin ?? 0.6,
+    );
+    gl.uniform1f(
+      stateUpdateLocations.smoothstepMax,
+      options.segmentationOptions?.smoothstepMax ?? 0.9,
+    );
 
     gl.uniform1i(
       stateUpdateLocations.selfieModel,

package/src/types.ts

@@ -8,11 +8,44 @@ export type BackgroundSource = {
 export type BackgroundFilter = 'blur' | 'image';
 export type BackgroundBlurLevel = 'low' | 'medium' | 'high' | number;
 
+/**
+ * Options for controlling the segmentation mask smoothing.
+ * These values are passed to the WebGL shader that blends
+ * consecutive segmentation masks over time.
+ */
+export interface SegmentationOptions {
+  /**
+   * Controls how fast the mask adapts to new segmentation results.
+   * Higher values make the mask react faster but may cause flickering.
+   * Lower values produce smoother transitions but increase latency.
+   * Value should be between 0 and 1.
+   * @default 0.8
+   */
+  smoothingFactor?: number;
+
+  /**
+   * Lower edge of the smoothstep function applied to the confidence mask.
+   * Confidence values below this are mapped to 0 (background).
+   * Value should be between 0 and 1, and less than smoothstepMax.
+   * @default 0.6
+   */
+  smoothstepMin?: number;
+
+  /**
+   * Upper edge of the smoothstep function applied to the confidence mask.
+   * Confidence values above this are mapped to 1 (foreground).
+   * Value should be between 0 and 1, and greater than smoothstepMin.
+   * @default 0.9
+   */
+  smoothstepMax?: number;
+}
+
 export interface SegmenterOptions {
   backgroundSource?: BackgroundSource | null;
   bgBlur: number;
   bgBlurRadius: number;
   isSelfieMode: boolean;
+  segmentationOptions?: SegmentationOptions;
 }
 /**
  * Static configuration for the processor, defining which background
@@ -24,6 +57,7 @@ export interface BackgroundOptions {
   backgroundFilter?: BackgroundFilter;
   backgroundBlurLevel?: BackgroundBlurLevel;
   backgroundImage?: string | undefined;
+  segmentationOptions?: SegmentationOptions;
 }
 
 /**
@@ -43,20 +77,11 @@ export interface VideoTrackProcessorHooks {
   onStats?: (stats: PerformanceStats) => void;
 }
 
-export const BACKGROUND_BLUR_MAP: Record<
-  BackgroundBlurLevel,
-  { bgBlur: number; bgBlurRadius: number }
-> = {
-  low: {
-    bgBlur: 5,
-    bgBlurRadius: 3,
-  },
-  medium: {
-    bgBlur: 15,
-    bgBlurRadius: 4,
-  },
-  high: {
-    bgBlur: 30,
-    bgBlurRadius: 5,
-  },
+/**
+ * Maps blur level to blur strength values.
+ */
+export const BACKGROUND_BLUR_MAP: Record<'low' | 'medium' | 'high', number> = {
+  low: 3,
+  medium: 5,
+  high: 7,
 };