@stream-io/video-filters-web 0.5.1 → 0.7.0

package/dist/src/BaseVideoProcessor.d.ts

@@ -0,0 +1,38 @@
+import { VideoTrackProcessorHooks } from './types';
+import { MediaStreamTrackGenerator } from './FallbackGenerator';
+import { MediaStreamTrackProcessor } from './FallbackProcessor';
+/**
+ * Base class for real-time video filters.
+ *
+ * It sets up the full pipeline that reads frames from the input track,
+ * processes them, and outputs a new track with your effect applied. Subclasses
+ * only need to implement `initialize` (run once before processing starts) and
+ * `transform` (called for every frame).
+ *
+ * Everything else—canvas setup, performance tracking, error handling, and
+ * clean shutdown is handled for you. Calling `start()` returns a processed
+ * `MediaStreamTrack` ready to use.
+ */
+export declare abstract class BaseVideoProcessor {
+    protected readonly track: MediaStreamVideoTrack;
+    protected readonly processor: MediaStreamTrackProcessor<VideoFrame>;
+    protected readonly generator: MediaStreamTrackGenerator<VideoFrame>;
+    protected readonly hooks: VideoTrackProcessorHooks;
+    protected readonly abortController: AbortController;
+    protected canvas: OffscreenCanvas;
+    private frames;
+    private delayTotal;
+    private lastStatsTime;
+    /**
+     * Constructs a new instance.
+     */
+    protected constructor(track: MediaStreamVideoTrack, hooks?: VideoTrackProcessorHooks);
+    start(): Promise<MediaStreamTrack>;
+    stop(): void;
+    private updateStats;
+    protected abstract initialize(): Promise<void>;
+    protected abstract transform(frame: VideoFrame): Promise<VideoFrame>;
+    protected onFlush(): void;
+    protected onStop(): void;
+    protected get processorName(): string;
+}

package/dist/src/FullScreenBlur.d.ts

@@ -0,0 +1,28 @@
+import { VideoTrackProcessorHooks } from './types';
+import { BaseVideoProcessor } from './BaseVideoProcessor';
+export interface FullScreenBlurOptions {
+    blurRadius?: number;
+}
+/**
+ * A video filter that applies a full-screen blur to each frame.
+ *
+ * It uses a WebGL renderer to blur the incoming camera track and outputs
+ * a new track with the effect applied. Setup and frame handling are managed
+ * by the base processor.
+ */
+export declare class FullScreenBlur extends BaseVideoProcessor {
+    private blurRenderer;
+    private readonly blurRadius;
+    /**
+     * Creates a new full-screen blur processor for the given video track.
+     *
+     * @param track - The input camera track to blur.
+     * @param options - Optional settings such as the blur radius.
+     * @param hooks - Optional callbacks for stats and error reporting.
+     */
+    constructor(track: MediaStreamVideoTrack, options?: FullScreenBlurOptions, hooks?: VideoTrackProcessorHooks);
+    protected initialize(): Promise<void>;
+    protected transform(frame: VideoFrame): Promise<VideoFrame>;
+    protected onStop(): void;
+    protected get processorName(): string;
+}

package/dist/src/FullScreenBlurRenderer.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Simple WebGL renderer for full-screen Gaussian blur.
+ * Uses a two-pass separable Gaussian blur (horizontal then vertical).
+ * Optimized for moderation use cases by blurring at reduced resolution (15% scale)
+ * and upscaling back to full resolution for output.
+ */
+export declare class FullScreenBlurRenderer {
+    readonly canvas: OffscreenCanvas;
+    readonly gl: WebGL2RenderingContext;
+    readonly blurProgramHandle: WebGLProgram;
+    readonly blurLocations: {
+        positionLocation: number;
+        texCoordLocation: number;
+        imageLocation: WebGLUniformLocation | null;
+        texelSizeLocation: WebGLUniformLocation | null;
+        directionLocation: WebGLUniformLocation | null;
+        weightsLocation: WebGLUniformLocation | null;
+    };
+    readonly passthroughProgramHandle: WebGLProgram;
+    readonly passthroughLocations: {
+        positionLocation: number;
+        texCoordLocation: number;
+        imageLocation: WebGLUniformLocation | null;
+    };
+    readonly positionBuffer: WebGLBuffer | null;
+    readonly texCoordBuffer: WebGLBuffer | null;
+    readonly pingTexture: WebGLTexture | null;
+    readonly pongTexture: WebGLTexture | null;
+    readonly pingFbo: WebGLFramebuffer | null;
+    readonly pongFbo: WebGLFramebuffer | null;
+    private inputTexture;
+    private isRunning;
+    private targetWidth;
+    private targetHeight;
+    private weightCache;
+    constructor(canvas: OffscreenCanvas);
+    private createAndLinkProgram;
+    private createShader;
+    private getGaussianWeights;
+    render(frame: VideoFrame, radius: number): void;
+    close(): void;
+}

package/dist/src/VirtualBackground.d.ts

@@ -1,42 +1,29 @@
 import { BackgroundOptions, VideoTrackProcessorHooks } from './types';
+import { BaseVideoProcessor } from './BaseVideoProcessor';
 /**
  * Wraps a video MediaStreamTrack in a real-time processing pipeline.
  * Incoming frames are processed through a transformer and re-emitted
  * on a new MediaStreamVideoTrack for downstream consumption.
  */
-export declare class VirtualBackground {
-    private readonly track;
+export declare class VirtualBackground extends BaseVideoProcessor {
     private readonly options;
-    private readonly hooks;
-    private readonly processor;
-    private readonly generator;
-    private canvas;
     private segmenter;
     private isSegmenterReady;
     private webGlRenderer;
-    private abortController;
-    private segmenterDelayTotal;
-    private frames;
-    private lastStatsTime;
+    private opts;
+    private latestCategoryMask;
+    private latestConfidenceMask;
+    private lastFrameTime;
+    private count;
     constructor(track: MediaStreamVideoTrack, options?: BackgroundOptions, hooks?: VideoTrackProcessorHooks);
-    start(): Promise<MediaStreamTrack>;
-    /**
-     * Loads and initializes the MediaPipe `ImageSegmenter`.
-     */
+    protected initialize(): Promise<void>;
     private initializeSegmenter;
-    /**
-     * Processes a single video frame.
-     *
-     * Performs segmentation via MediaPipe and then composites the frame
-     * through the WebGL renderer to apply background effects.
-     *
-     * @param frame - The incoming frame from the processor.
-     * @param opts - The segmentation options to use.
-     *
-     * @returns A new `VideoFrame` containing the processed image.
-     */
-    private transform;
-    private loadBackground;
+    protected transform(frame: VideoFrame): Promise<VideoFrame>;
+    private runSegmentation;
     private initializeSegmenterOptions;
-    stop(): void;
+    private loadBackground;
+    protected onFlush(): void;
+    protected onStop(): void;
+    private destroySegmenter;
+    protected get processorName(): string;
 }

package/index.ts

@@ -5,3 +5,4 @@ export * from './src/legacy/tflite';
 export * from './src/mediapipe';
 export * from './src/types';
 export * from './src/VirtualBackground';
+export * from './src/FullScreenBlur';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-filters-web",
-  "version": "0.5.1",
+  "version": "0.7.0",
   "main": "./dist/index.cjs.js",
   "module": "./dist/index.es.js",
   "types": "./dist/index.d.ts",

package/src/BaseVideoProcessor.ts

@@ -0,0 +1,132 @@
+import { VideoTrackProcessorHooks } from './types';
+import { TrackGenerator, MediaStreamTrackGenerator } from './FallbackGenerator';
+import { MediaStreamTrackProcessor, TrackProcessor } from './FallbackProcessor';
+
+/**
+ * Base class for real-time video filters.
+ *
+ * It sets up the full pipeline that reads frames from the input track,
+ * processes them, and outputs a new track with your effect applied. Subclasses
+ * only need to implement `initialize` (run once before processing starts) and
+ * `transform` (called for every frame).
+ *
+ * Everything else—canvas setup, performance tracking, error handling, and
+ * clean shutdown is handled for you. Calling `start()` returns a processed
+ * `MediaStreamTrack` ready to use.
+ */
+export abstract class BaseVideoProcessor {
+  protected readonly processor: MediaStreamTrackProcessor<VideoFrame>;
+  protected readonly generator: MediaStreamTrackGenerator<VideoFrame>;
+
+  protected readonly hooks: VideoTrackProcessorHooks;
+
+  protected readonly abortController = new AbortController();
+  protected canvas!: OffscreenCanvas;
+
+  private frames = 0;
+  private delayTotal = 0;
+  private lastStatsTime = 0;
+
+  /**
+   * Constructs a new instance.
+   */
+  protected constructor(
+    protected readonly track: MediaStreamVideoTrack,
+    hooks: VideoTrackProcessorHooks = {},
+  ) {
+    this.processor = new TrackProcessor({ track });
+    this.generator = new TrackGenerator({
+      kind: 'video',
+      signalTarget: track,
+    });
+    this.hooks = hooks;
+  }
+
+  public async start(): Promise<MediaStreamTrack> {
+    const { readable } = this.processor;
+    const { writable } = this.generator;
+
+    const { width = 1280, height = 720 } = this.track.getSettings();
+    this.canvas = new OffscreenCanvas(width, height);
+
+    await this.initialize();
+
+    const transformStream = new TransformStream<VideoFrame, VideoFrame>({
+      transform: async (frame, controller) => {
+        try {
+          if (this.abortController.signal.aborted) return frame.close();
+
+          if (
+            this.canvas.width !== frame.displayWidth ||
+            this.canvas.height !== frame.displayHeight
+          ) {
+            this.canvas.width = frame.displayWidth;
+            this.canvas.height = frame.displayHeight;
+          }
+
+          const start = performance.now();
+          const processed = await this.transform(frame);
+          const delay = performance.now() - start;
+
+          this.updateStats(delay);
+          controller.enqueue(processed);
+        } catch (e) {
+          this.hooks.onError?.(e);
+        } finally {
+          frame.close();
+        }
+      },
+      flush: () => this.onFlush(),
+    });
+
+    readable
+      .pipeThrough(transformStream, { signal: this.abortController.signal })
+      .pipeTo(writable, { signal: this.abortController.signal })
+      .catch((e) => {
+        if (e.name !== 'AbortError' && e.name !== 'InvalidStateError') {
+          console.error(`[${this.processorName}] Error processing track:`, e);
+          this.hooks.onError?.(e);
+        }
+      });
+
+    return this.generator;
+  }
+
+  public stop(): void {
+    this.abortController.abort();
+    this.generator.stop();
+    this.onStop();
+  }
+
+  private updateStats(delay: number): void {
+    this.frames++;
+    this.delayTotal += delay;
+
+    const now = performance.now();
+    if (this.lastStatsTime === 0) {
+      this.lastStatsTime = now;
+      return;
+    }
+
+    if (now - this.lastStatsTime >= 1000) {
+      const avgDelay = Math.round((this.delayTotal / this.frames) * 100) / 100;
+      const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
+
+      this.hooks.onStats?.({ delay: avgDelay, fps, timestamp: now });
+
+      this.frames = 0;
+      this.delayTotal = 0;
+      this.lastStatsTime = now;
+    }
+  }
+
+  protected abstract initialize(): Promise<void>;
+  protected abstract transform(frame: VideoFrame): Promise<VideoFrame>;
+
+  protected onFlush(): void {}
+  protected onStop(): void {}
+
+  protected get processorName(): string {
+    return 'base-processor';
+  }
+}

package/src/FallbackProcessor.ts

@@ -49,7 +49,6 @@ class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
     let timestamp = 0;
     const frameRate = track.getSettings().frameRate || 30;
     let frameDuration = 1000 / frameRate;
-    let lastVideoTime = -1;
 
     this.workerTimer = new WorkerTimer({ useWorker: true });
     this.readable = new ReadableStream({
@@ -77,18 +76,6 @@ class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
         }
         timestamp = performance.now();
 
-        const currentTime = this.video.currentTime;
-        const hasNewFrame = currentTime !== lastVideoTime;
-
-        if (!hasNewFrame) {
-          await new Promise((r: (value?: unknown) => void) =>
-            this.workerTimer.setTimeout(r, frameDuration),
-          );
-          return;
-        }
-
-        lastVideoTime = currentTime;
-
         if (
           canvas.width !== this.video.videoWidth ||
           canvas.height !== this.video.videoHeight
@@ -100,7 +87,10 @@ class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
         ctx.drawImage(this.video, 0, 0);
 
         try {
-          const frame = new VideoFrame(canvas, { timestamp });
+          const frame = new VideoFrame(canvas, {
+            timestamp: Math.round(this.video.currentTime * 1000000),
+          });
+
           controller.enqueue(frame);
         } catch (err) {
           running = false;

package/src/FullScreenBlur.ts

@@ -0,0 +1,52 @@
+import { VideoTrackProcessorHooks } from './types';
+import { BaseVideoProcessor } from './BaseVideoProcessor';
+import { FullScreenBlurRenderer } from './FullScreenBlurRenderer';
+
+export interface FullScreenBlurOptions {
+  blurRadius?: number;
+}
+
+/**
+ * A video filter that applies a full-screen blur to each frame.
+ *
+ * It uses a WebGL renderer to blur the incoming camera track and outputs
+ * a new track with the effect applied. Setup and frame handling are managed
+ * by the base processor.
+ */
+export class FullScreenBlur extends BaseVideoProcessor {
+  private blurRenderer!: FullScreenBlurRenderer;
+  private readonly blurRadius: number;
+
+  /**
+   * Creates a new full-screen blur processor for the given video track.
+   *
+   * @param track - The input camera track to blur.
+   * @param options - Optional settings such as the blur radius.
+   * @param hooks - Optional callbacks for stats and error reporting.
+   */
+  constructor(
+    track: MediaStreamVideoTrack,
+    options: FullScreenBlurOptions = {},
+    hooks: VideoTrackProcessorHooks = {},
+  ) {
+    super(track, hooks);
+    this.blurRadius = options.blurRadius ?? 6;
+  }
+
+  protected async initialize(): Promise<void> {
+    this.blurRenderer = new FullScreenBlurRenderer(this.canvas);
+  }
+
+  protected async transform(frame: VideoFrame): Promise<VideoFrame> {
+    this.blurRenderer.render(frame, this.blurRadius);
+    return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
+  }
+
+  protected onStop(): void {
+    this.blurRenderer?.close();
+  }
+
+  protected get processorName(): string {
+    return 'fullscreen-blur';
+  }
+}