@stream-io/video-filters-web 0.3.0 → 0.5.0

package/dist/src/FallbackGenerator.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Type representing a video track generator that can be either the native
+ * MediaStreamTrackGenerator or the fallback implementation.
+ */
+export interface MediaStreamTrackGenerator<T> extends MediaStreamTrack {
+    writable: WritableStream<T>;
+}
+/**
+ * Configuration options for creating a track generator.
+ */
+export interface TrackGeneratorOptions {
+    readonly kind: 'video';
+    readonly signalTarget?: MediaStreamTrack;
+}
+type VideoTrackGeneratorConstructor = new (options: TrackGeneratorOptions) => MediaStreamTrackGenerator<VideoFrame>;
+declare const TrackGenerator: VideoTrackGeneratorConstructor;
+export { TrackGenerator };

package/dist/src/FallbackProcessor.d.ts

@@ -0,0 +1,38 @@
+import { WorkerTimer } from '@stream-io/worker-timer';
+/**
+ * Representing a video track processor that can be either the native
+ * MediaStreamTrackProcessor or the fallback implementation.
+ */
+export interface MediaStreamTrackProcessor<T> {
+    readonly readable: ReadableStream<T>;
+}
+/**
+ * Configuration options for creating a track processor.
+ */
+export interface TrackProcessorOptions {
+    readonly track: MediaStreamTrack;
+}
+/**
+ * Fallback implementation for browsers without MediaStreamTrackGenerator.
+ *
+ * Produces a video MediaStreamTrack sourced from a canvas and exposes a
+ * WritableStream<VideoFrame> on track.writable. Written frames are drawn
+ * into the canvas and update the underlying track automatically.
+ */
+declare class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
+    readonly readable: ReadableStream<VideoFrame>;
+    readonly workerTimer: WorkerTimer;
+    readonly video: HTMLVideoElement;
+    constructor({ track }: TrackProcessorOptions);
+    close: () => void;
+}
+declare const TrackProcessor: typeof FallbackProcessor | {
+    new (init: MediaStreamTrackProcessorInit & {
+        track: MediaStreamAudioTrack;
+    }): globalThis.MediaStreamTrackProcessor<AudioData>;
+    new (init: MediaStreamTrackProcessorInit & {
+        track: MediaStreamVideoTrack;
+    }): globalThis.MediaStreamTrackProcessor<VideoFrame>;
+    prototype: globalThis.MediaStreamTrackProcessor<any>;
+};
+export { TrackProcessor };

package/dist/src/VirtualBackground.d.ts

@@ -0,0 +1,42 @@
+import { BackgroundOptions, VideoTrackProcessorHooks } from './types';
+/**
+ * Wraps a video MediaStreamTrack in a real-time processing pipeline.
+ * Incoming frames are processed through a transformer and re-emitted
+ * on a new MediaStreamVideoTrack for downstream consumption.
+ */
+export declare class VirtualBackground {
+    private readonly track;
+    private readonly options;
+    private readonly hooks;
+    private readonly processor;
+    private readonly generator;
+    private canvas;
+    private segmenter;
+    private isSegmenterReady;
+    private webGlRenderer;
+    private abortController;
+    private segmenterDelayTotal;
+    private frames;
+    private lastStatsTime;
+    constructor(track: MediaStreamVideoTrack, options?: BackgroundOptions, hooks?: VideoTrackProcessorHooks);
+    start(): Promise<MediaStreamTrack>;
+    /**
+     * Loads and initializes the MediaPipe `ImageSegmenter`.
+     */
+    private initializeSegmenter;
+    /**
+     * Processes a single video frame.
+     *
+     * Performs segmentation via MediaPipe and then composites the frame
+     * through the WebGL renderer to apply background effects.
+     *
+     * @param frame - The incoming frame from the processor.
+     * @param opts - The segmentation options to use.
+     *
+     * @returns A new `VideoFrame` containing the processed image.
+     */
+    private transform;
+    private loadBackground;
+    private initializeSegmenterOptions;
+    stop(): void;
+}

package/dist/src/WebGLRenderer.d.ts

@@ -0,0 +1,80 @@
+import { BackgroundSource } from './types';
+export declare class WebGLRenderer {
+    readonly canvas: OffscreenCanvas;
+    readonly gl: WebGL2RenderingContext;
+    readonly stateUpdateProgram: WebGLProgram;
+    readonly maskRefineProgram: WebGLProgram;
+    readonly blurProgram: WebGLProgram;
+    readonly blendProgram: WebGLProgram;
+    readonly stateUpdateLocations: {
+        position: number;
+        texCoord: number;
+        categoryTexture: WebGLUniformLocation | null;
+        confidenceTexture: WebGLUniformLocation | null;
+        prevStateTexture: WebGLUniformLocation | null;
+        smoothingFactor: WebGLUniformLocation | null;
+        smoothstepMin: WebGLUniformLocation | null;
+        smoothstepMax: WebGLUniformLocation | null;
+        selfieModel: WebGLUniformLocation | null;
+    };
+    readonly maskRefineLocations: {
+        position: number;
+        texCoord: number;
+        maskTexture: WebGLUniformLocation | null;
+        frameTexture: WebGLUniformLocation | null;
+        texelSize: WebGLUniformLocation | null;
+        sigmaSpatial: WebGLUniformLocation | null;
+        sigmaRange: WebGLUniformLocation | null;
+    };
+    readonly blurLocations: {
+        position: number;
+        texCoord: number;
+        image: WebGLUniformLocation | null;
+        texelSize: WebGLUniformLocation | null;
+        sigma: WebGLUniformLocation | null;
+        radiusScale: WebGLUniformLocation | null;
+        personMask: WebGLUniformLocation | null;
+        direction: WebGLUniformLocation | null;
+    };
+    readonly blendLocations: {
+        position: number;
+        texCoord: number;
+        frameTexture: WebGLUniformLocation | null;
+        currentStateTexture: WebGLUniformLocation | null;
+        backgroundTexture: WebGLUniformLocation | null;
+        bgImageDimensions: WebGLUniformLocation | null;
+        canvasDimensions: WebGLUniformLocation | null;
+        borderSmooth: WebGLUniformLocation | null;
+        bgBlur: WebGLUniformLocation | null;
+        bgBlurRadius: WebGLUniformLocation | null;
+        enabled: WebGLUniformLocation | null;
+    };
+    readonly positionBuffer: WebGLBuffer | null;
+    readonly texCoordBuffer: WebGLBuffer | null;
+    readonly storedStateTextures: (WebGLTexture | null)[];
+    readonly fbo: WebGLFramebuffer | null;
+    readonly refineFbo: WebGLFramebuffer | null;
+    readonly refinedMaskTexture: WebGLTexture | null;
+    readonly frameTexture: WebGLTexture | null;
+    readonly blurTexture1: WebGLTexture | null;
+    readonly blurTexture2: WebGLTexture | null;
+    readonly blurFbo1: WebGLFramebuffer | null;
+    readonly blurFbo2: WebGLFramebuffer | null;
+    private running;
+    private static readonly DEFAULT_BG_COLOR;
+    private currentStateIndex;
+    private backgroundRenderInfo;
+    private activeBackgroundSourceIdentifier;
+    constructor(canvas: OffscreenCanvas);
+    private createAndLinkProgram;
+    private createShader;
+    private createColorTexture;
+    private updateBackgroundIfNeeded;
+    render(videoFrame: VideoFrame, options: {
+        backgroundSource?: BackgroundSource | null;
+        bgBlur: number;
+        bgBlurRadius: number;
+        isSelfieMode: boolean;
+    }, categoryTexture?: WebGLTexture, confidenceTexture?: WebGLTexture): void;
+    close(): void;
+}

package/dist/src/compatibility.d.ts

@@ -14,3 +14,8 @@ export type PlatformSupportFlags = {
  * the necessary APIs required for the video filters.
  */
 export declare const isPlatformSupported: ({ forceMobileSupport, forceSafariSupport, }?: PlatformSupportFlags) => Promise<boolean>;
+/**
+ * Runs a check to see if the current platform supports
+ * the necessary APIs required for the MediaPipe-based video filters.
+ */
+export declare const isMediaPipePlatformSupported: ({ forceMobileSupport, forceSafariSupport, }?: PlatformSupportFlags) => Promise<boolean>;

package/dist/src/{createRenderer.d.ts → legacy/createRenderer.d.ts}

@@ -1,7 +1,6 @@
 import { TFLite } from './tflite';
 import { SegmentationLevel } from './segmentation';
-export type BackgroundFilter = 'blur' | 'image';
-export type BackgroundBlurLevel = 'low' | 'medium' | 'high' | number;
+import { BackgroundBlurLevel, BackgroundFilter } from '../types';
 export type Renderer = {
     /**
      * Disposes of the renderer.

package/dist/src/{webgl2 → legacy/webgl2}/backgroundBlurStage.d.ts

@@ -1,4 +1,4 @@
-import type { BackgroundBlurLevel } from '../createRenderer';
+import { BackgroundBlurLevel } from '../../types';
 export type BackgroundBlurStage = {
     render(): void;
     updateCoverage(coverage: [number, number]): void;

package/dist/src/{webgl2 → legacy/webgl2}/webgl2Pipeline.d.ts

@@ -1,6 +1,6 @@
 import { TFLite } from '../tflite';
-import { BackgroundBlurLevel, BackgroundFilter } from '../createRenderer';
 import { SegmentationParams } from '../segmentation';
+import type { BackgroundBlurLevel, BackgroundFilter } from '../../types';
 export declare function buildWebGL2Pipeline(videoSource: HTMLVideoElement, backgroundImage: HTMLImageElement | undefined, blurLevel: BackgroundBlurLevel | undefined, backgroundFilter: BackgroundFilter, canvas: HTMLCanvasElement, tflite: TFLite, segmentationConfig: SegmentationParams, onError?: (error: any) => void): {
     render: () => void;
     updatePostProcessingConfig: () => void;

package/dist/src/mediapipe.d.ts

@@ -0,0 +1,4 @@
+export declare const loadMediaPipe: (options?: {
+    basePath?: string;
+    modelPath?: string;
+}) => Promise<ArrayBuffer>;

package/dist/src/types.d.ts

@@ -0,0 +1,45 @@
+export type BackgroundSource = {
+    type: string;
+    media?: ImageBitmap | ReadableStream;
+    url: string;
+    video?: HTMLVideoElement;
+    track?: MediaStreamTrack;
+};
+export type BackgroundFilter = 'blur' | 'image';
+export type BackgroundBlurLevel = 'low' | 'medium' | 'high' | number;
+export interface SegmenterOptions {
+    backgroundSource?: BackgroundSource | null;
+    bgBlur: number;
+    bgBlurRadius: number;
+    isSelfieMode: boolean;
+}
+/**
+ * Static configuration for the processor, defining which background
+ * effect should be applied and how it should behave.
+ */
+export interface BackgroundOptions {
+    basePath?: string;
+    modelPath?: string;
+    backgroundFilter?: BackgroundFilter;
+    backgroundBlurLevel?: BackgroundBlurLevel;
+    backgroundImage?: string | undefined;
+}
+/**
+ * Performance statistics for video processing.
+ */
+export interface PerformanceStats {
+    delay: number;
+    fps: number;
+    timestamp: number;
+}
+/**
+ * Runtime hooks for handling lifecycle or error events.
+ */
+export interface VideoTrackProcessorHooks {
+    onError?: (error: unknown) => void;
+    onStats?: (stats: PerformanceStats) => void;
+}
+export declare const BACKGROUND_BLUR_MAP: Record<BackgroundBlurLevel, {
+    bgBlur: number;
+    bgBlurRadius: number;
+}>;

package/index.ts

@@ -1,4 +1,7 @@
 export * from './src/compatibility';
-export * from './src/createRenderer';
-export { SegmentationLevel } from './src/segmentation';
-export * from './src/tflite';
+export * from './src/legacy/createRenderer';
+export { SegmentationLevel } from './src/legacy/segmentation';
+export * from './src/legacy/tflite';
+export * from './src/mediapipe';
+export * from './src/types';
+export * from './src/VirtualBackground';