@stream-io/video-filters-web 0.3.0 → 0.5.0

package/src/FallbackGenerator.ts

@@ -0,0 +1,90 @@
+/**
+ * Type representing a video track generator that can be either the native
+ * MediaStreamTrackGenerator or the fallback implementation.
+ */
+export interface MediaStreamTrackGenerator<T> extends MediaStreamTrack {
+  writable: WritableStream<T>;
+}
+
+/**
+ * Configuration options for creating a track generator.
+ */
+export interface TrackGeneratorOptions {
+  readonly kind: 'video';
+  readonly signalTarget?: MediaStreamTrack;
+}
+
+/**
+ * Fallback video processor for browsers that do not support MediaStreamTrackGenerator.
+ *
+ * Produces a video MediaStreamTrack sourced from a canvas and exposes
+ * a WritableStream<VideoFrame> on track.writable for writing frames.
+ */
+class FallbackGenerator {
+  constructor({ kind, signalTarget }: TrackGeneratorOptions) {
+    if (kind !== 'video') {
+      throw new Error('Only video tracks are supported');
+    }
+
+    const canvas = document.createElement('canvas');
+    const ctx = canvas.getContext('2d', { desynchronized: true });
+    if (!ctx) {
+      throw new Error('Failed to get 2D context from canvas');
+    }
+
+    const mediaStream = canvas.captureStream();
+    const track =
+      mediaStream.getVideoTracks()[0] as MediaStreamTrackGenerator<VideoFrame>;
+
+    const height = signalTarget?.getSettings().height;
+    const width = signalTarget?.getSettings().width;
+    if (height && width) {
+      canvas.height = height;
+      canvas.width = width;
+    }
+
+    if (!track) {
+      throw new Error('Failed to create canvas track');
+    }
+
+    if (signalTarget) {
+      signalTarget.addEventListener('ended', () => {
+        track.stop();
+      });
+    }
+
+    track.writable = new WritableStream({
+      write: (frame: VideoFrame) => {
+        if (
+          canvas.width !== frame.displayWidth ||
+          canvas.height !== frame.displayHeight
+        ) {
+          canvas.width = frame.displayWidth;
+          canvas.height = frame.displayHeight;
+        }
+
+        ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+        frame.close();
+      },
+      abort: () => {
+        track.stop();
+      },
+      close: () => {
+        track.stop();
+      },
+    });
+
+    return track as MediaStreamTrackGenerator<VideoFrame>;
+  }
+}
+
+type VideoTrackGeneratorConstructor = new (
+  options: TrackGeneratorOptions,
+) => MediaStreamTrackGenerator<VideoFrame>;
+
+const TrackGenerator: VideoTrackGeneratorConstructor =
+  typeof MediaStreamTrackGenerator !== 'undefined'
+    ? (MediaStreamTrackGenerator as unknown as VideoTrackGeneratorConstructor)
+    : (FallbackGenerator as unknown as VideoTrackGeneratorConstructor);
+
+export { TrackGenerator };

package/src/FallbackProcessor.ts

@@ -0,0 +1,132 @@
+import { WorkerTimer } from '@stream-io/worker-timer';
+
+/**
+ * Representing a video track processor that can be either the native
+ * MediaStreamTrackProcessor or the fallback implementation.
+ */
+export interface MediaStreamTrackProcessor<T> {
+  readonly readable: ReadableStream<T>;
+}
+
+/**
+ * Configuration options for creating a track processor.
+ */
+export interface TrackProcessorOptions {
+  readonly track: MediaStreamTrack;
+}
+
+/**
+ * Fallback implementation for browsers without MediaStreamTrackGenerator.
+ *
+ * Produces a video MediaStreamTrack sourced from a canvas and exposes a
+ * WritableStream<VideoFrame> on track.writable. Written frames are drawn
+ * into the canvas and update the underlying track automatically.
+ */
+class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
+  readonly readable: ReadableStream<VideoFrame>;
+
+  readonly workerTimer: WorkerTimer;
+  readonly video: HTMLVideoElement;
+
+  constructor({ track }: TrackProcessorOptions) {
+    if (!track) throw new Error('MediaStreamTrack is required');
+    if (track.kind !== 'video') {
+      throw new Error('MediaStreamTrack must be video');
+    }
+    let running = true;
+
+    this.video = document.createElement('video');
+
+    this.video.muted = true;
+    this.video.playsInline = true;
+    this.video.srcObject = new MediaStream([track]);
+
+    const canvas = new OffscreenCanvas(1, 1);
+    const ctx = canvas.getContext('2d');
+
+    if (!ctx) throw new Error('Failed to get 2D context from OffscreenCanvas');
+
+    let timestamp = 0;
+    const frameRate = track.getSettings().frameRate || 30;
+    let frameDuration = 1000 / frameRate;
+    let lastVideoTime = -1;
+
+    this.workerTimer = new WorkerTimer({ useWorker: true });
+    this.readable = new ReadableStream({
+      start: async () => {
+        await Promise.all([
+          this.video.play(),
+          new Promise((r) =>
+            this.video.addEventListener('loadeddata', r, { once: true }),
+          ),
+        ]);
+        frameDuration = 1000 / (track.getSettings().frameRate || 30);
+        timestamp = performance.now();
+      },
+      pull: async (controller) => {
+        if (!running) {
+          controller.close();
+          this.close();
+          return;
+        }
+        const delta = performance.now() - timestamp;
+        if (delta <= frameDuration) {
+          await new Promise((r: (value?: unknown) => void) =>
+            this.workerTimer.setTimeout(r, frameDuration - delta),
+          );
+        }
+        timestamp = performance.now();
+
+        const currentTime = this.video.currentTime;
+        const hasNewFrame = currentTime !== lastVideoTime;
+
+        if (!hasNewFrame) {
+          await new Promise((r: (value?: unknown) => void) =>
+            this.workerTimer.setTimeout(r, frameDuration),
+          );
+          return;
+        }
+
+        lastVideoTime = currentTime;
+
+        if (
+          canvas.width !== this.video.videoWidth ||
+          canvas.height !== this.video.videoHeight
+        ) {
+          canvas.width = this.video.videoWidth;
+          canvas.height = this.video.videoHeight;
+        }
+
+        ctx.drawImage(this.video, 0, 0);
+
+        try {
+          const frame = new VideoFrame(canvas, { timestamp });
+          controller.enqueue(frame);
+        } catch (err) {
+          running = false;
+          controller.error(err);
+          this.close();
+        }
+      },
+      cancel: () => {
+        running = false;
+        this.close();
+      },
+    });
+  }
+
+  public close = () => {
+    this.video.pause();
+    this.video.srcObject = null;
+    this.video.src = '';
+
+    this.workerTimer.destroy();
+  };
+}
+
+const TrackProcessor =
+  typeof MediaStreamTrackProcessor !== 'undefined'
+    ? MediaStreamTrackProcessor
+    : FallbackProcessor;
+
+export { TrackProcessor };

package/src/VirtualBackground.ts

@@ -0,0 +1,276 @@
+import {
+  BACKGROUND_BLUR_MAP,
+  BackgroundOptions,
+  SegmenterOptions,
+  VideoTrackProcessorHooks,
+} from './types';
+import { FilesetResolver, ImageSegmenter } from '@mediapipe/tasks-vision';
+import { WebGLRenderer } from './WebGLRenderer';
+import { packageName, version } from './version';
+import { TrackGenerator, MediaStreamTrackGenerator } from './FallbackGenerator';
+import { MediaStreamTrackProcessor, TrackProcessor } from './FallbackProcessor';
+
+/**
+ * Wraps a video MediaStreamTrack in a real-time processing pipeline.
+ * Incoming frames are processed through a transformer and re-emitted
+ * on a new MediaStreamVideoTrack for downstream consumption.
+ */
+export class VirtualBackground {
+  private readonly processor: MediaStreamTrackProcessor<VideoFrame>;
+  private readonly generator: MediaStreamTrackGenerator<VideoFrame>;
+
+  private canvas!: OffscreenCanvas;
+  private segmenter: ImageSegmenter | null = null;
+  private isSegmenterReady = false;
+  private webGlRenderer!: WebGLRenderer;
+  private abortController: AbortController;
+
+  private segmenterDelayTotal = 0;
+  private frames = 0;
+  private lastStatsTime = 0;
+
+  constructor(
+    private readonly track: MediaStreamVideoTrack,
+    private readonly options: BackgroundOptions = {},
+    private readonly hooks: VideoTrackProcessorHooks = {},
+  ) {
+    this.processor = new TrackProcessor({ track });
+    this.generator = new TrackGenerator({
+      kind: 'video',
+      signalTarget: track,
+    });
+
+    this.abortController = new AbortController();
+  }
+
+  public async start(): Promise<MediaStreamTrack> {
+    const { onError } = this.hooks;
+
+    const { readable } = this.processor;
+    const { writable } = this.generator;
+
+    const displayWidth = this.track.getSettings().width ?? 1280;
+    const displayHeight = this.track.getSettings().height ?? 720;
+
+    this.canvas = new OffscreenCanvas(displayWidth, displayHeight);
+    this.webGlRenderer = new WebGLRenderer(this.canvas);
+
+    await this.initializeSegmenter();
+
+    const opts = await this.initializeSegmenterOptions();
+
+    const transformStream = new TransformStream<VideoFrame, VideoFrame>({
+      transform: async (frame, controller) => {
+        try {
+          if (this.abortController.signal.aborted) {
+            return frame.close();
+          }
+
+          const processed = await this.transform(frame, opts);
+          controller.enqueue(processed);
+        } catch (e) {
+          console.error('[virtual-background] error processing frame:', e);
+          this.hooks.onError?.(e);
+
+          if (!this.abortController.signal.aborted) {
+            controller.enqueue(frame);
+          }
+        } finally {
+          frame.close();
+        }
+      },
+      flush: () => {
+        if (this.segmenter) {
+          this.segmenter.close();
+          this.segmenter = null;
+        }
+        this.isSegmenterReady = false;
+      },
+    });
+
+    const signal = this.abortController.signal;
+
+    readable
+      .pipeThrough(transformStream, { signal })
+      .pipeTo(writable)
+      .catch((e) => {
+        if (e.name !== 'AbortError') {
+          console.error('[virtual-background] Error processing track:', e);
+          onError?.(e);
+        }
+      });
+
+    return this.generator;
+  }
+
+  /**
+   * Loads and initializes the MediaPipe `ImageSegmenter`.
+   */
+  private async initializeSegmenter() {
+    try {
+      const basePath =
+        this.options?.basePath ||
+        `https://unpkg.com/${packageName}@${version}/mediapipe`;
+
+      const defaultModelPath = `${basePath}/models/selfie_segmenter.tflite`;
+
+      const model = this.options?.modelPath || defaultModelPath;
+
+      const wasmPath = `${basePath}/wasm`;
+
+      const fileset = await FilesetResolver.forVisionTasks(wasmPath);
+
+      this.segmenter = await ImageSegmenter.createFromOptions(fileset, {
+        baseOptions: {
+          modelAssetPath: model,
+          delegate: 'GPU',
+        },
+        runningMode: 'VIDEO',
+        outputCategoryMask: true,
+        outputConfidenceMasks: true,
+        canvas: this.canvas,
+      });
+
+      this.isSegmenterReady = true;
+    } catch (error) {
+      console.error(
+        '[virtual-background] Failed to initialize MediaPipe segmenter:',
+        error,
+      );
+      this.isSegmenterReady = false;
+    }
+  }
+
+  /**
+   * Processes a single video frame.
+   *
+   * Performs segmentation via MediaPipe and then composites the frame
+   * through the WebGL renderer to apply background effects.
+   *
+   * @param frame - The incoming frame from the processor.
+   * @param opts - The segmentation options to use.
+   *
+   * @returns A new `VideoFrame` containing the processed image.
+   */
+  private async transform(
+    frame: VideoFrame,
+    opts: SegmenterOptions,
+  ): Promise<VideoFrame> {
+    if (this.isSegmenterReady && this.segmenter) {
+      try {
+        const start = performance.now();
+        await new Promise<void>((resolve) => {
+          this.segmenter!.segmentForVideo(frame, frame.timestamp, (result) => {
+            const categoryMask = result.categoryMask!.getAsWebGLTexture();
+            const confidenceMask =
+              result.confidenceMasks![0].getAsWebGLTexture();
+
+            this.webGlRenderer.render(
+              frame,
+              opts,
+              categoryMask,
+              confidenceMask,
+            );
+
+            const now = performance.now();
+            this.segmenterDelayTotal += now - start;
+            this.frames++;
+
+            if (this.lastStatsTime === 0) {
+              this.lastStatsTime = now;
+            }
+
+            if (now - this.lastStatsTime > 1000) {
+              const delay =
+                Math.round((this.segmenterDelayTotal / this.frames) * 100) /
+                100;
+              const fps = Math.round(
+                (1000 * this.frames) / (now - this.lastStatsTime),
+              );
+
+              this.hooks.onStats?.({ delay, fps, timestamp: now });
+
+              this.lastStatsTime = now;
+              this.segmenterDelayTotal = 0;
+              this.frames = 0;
+            }
+
+            resolve();
+          });
+        });
+      } catch (error) {
+        console.error('[virtual-background] Error during segmentation:', error);
+      }
+    }
+
+    return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
+  }
+
+  private async loadBackground(url: string | undefined) {
+    if (!url) {
+      return;
+    }
+
+    const response = await fetch(url);
+    if (!response.ok) {
+      console.error(
+        `[virtual-background] Failed to fetch background source ${url} (status: ${response.status})`,
+      );
+      return;
+    }
+    const blob = await response.blob();
+
+    const imageBitmap = await createImageBitmap(blob);
+    return { type: 'image', media: imageBitmap, url };
+  }
+
+  private async initializeSegmenterOptions(): Promise<SegmenterOptions> {
+    const isSelfieMode = this.options.modelPath
+      ? this.options.modelPath?.includes('selfie_segmenter')
+      : true;
+
+    if (this.options.backgroundFilter === 'image') {
+      return {
+        backgroundSource: await this.loadBackground(
+          this.options.backgroundImage,
+        ),
+        bgBlur: 0,
+        bgBlurRadius: 0,
+        isSelfieMode,
+      };
+    }
+
+    const blurLevel = this.options.backgroundBlurLevel;
+    if (typeof blurLevel === 'string') {
+      return {
+        ...BACKGROUND_BLUR_MAP[blurLevel],
+        backgroundSource: undefined,
+        isSelfieMode,
+      };
+    }
+
+    const numeric = blurLevel ?? 5;
+
+    const bgBlur = Math.min(numeric * 3, 30);
+    const bgBlurRadius = Math.min(numeric, 10);
+
+    return {
+      backgroundSource: undefined,
+      bgBlur,
+      bgBlurRadius,
+      isSelfieMode,
+    };
+  }
+
+  public stop(): void {
+    this.abortController.abort();
+    this.webGlRenderer.close();
+    this.generator.stop();
+
+    if (this.segmenter) {
+      this.segmenter.close();
+      this.segmenter = null;
+    }
+    this.isSegmenterReady = false;
+  }
+}