@stream-io/video-filters-web 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs.js CHANGED
@@ -1561,7 +1561,7 @@ const createTFLiteSIMDModule = (__Module) => {
1561
1561
  return __Module.ready;
1562
1562
  };
1563
1563
 
1564
- const version = "0.5.0";
1564
+ const version = "0.6.0";
1565
1565
  const packageName = "@stream-io/video-filters-web";
1566
1566
 
1567
1567
  // @ts-expect-error - module is not declared
@@ -2405,7 +2405,6 @@ class FallbackProcessor {
2405
2405
  let timestamp = 0;
2406
2406
  const frameRate = track.getSettings().frameRate || 30;
2407
2407
  let frameDuration = 1000 / frameRate;
2408
- let lastVideoTime = -1;
2409
2408
  this.workerTimer = new workerTimer.WorkerTimer({ useWorker: true });
2410
2409
  this.readable = new ReadableStream({
2411
2410
  start: async () => {
@@ -2427,13 +2426,6 @@ class FallbackProcessor {
2427
2426
  await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration - delta));
2428
2427
  }
2429
2428
  timestamp = performance.now();
2430
- const currentTime = this.video.currentTime;
2431
- const hasNewFrame = currentTime !== lastVideoTime;
2432
- if (!hasNewFrame) {
2433
- await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration));
2434
- return;
2435
- }
2436
- lastVideoTime = currentTime;
2437
2429
  if (canvas.width !== this.video.videoWidth ||
2438
2430
  canvas.height !== this.video.videoHeight) {
2439
2431
  canvas.width = this.video.videoWidth;
@@ -2441,7 +2433,9 @@ class FallbackProcessor {
2441
2433
  }
2442
2434
  ctx.drawImage(this.video, 0, 0);
2443
2435
  try {
2444
- const frame = new VideoFrame(canvas, { timestamp });
2436
+ const frame = new VideoFrame(canvas, {
2437
+ timestamp: Math.round(this.video.currentTime * 1000000),
2438
+ });
2445
2439
  controller.enqueue(frame);
2446
2440
  }
2447
2441
  catch (err) {
@@ -2462,93 +2456,134 @@ const TrackProcessor = typeof MediaStreamTrackProcessor !== 'undefined'
2462
2456
  : FallbackProcessor;
2463
2457
 
2464
2458
  /**
2465
- * Wraps a video MediaStreamTrack in a real-time processing pipeline.
2466
- * Incoming frames are processed through a transformer and re-emitted
2467
- * on a new MediaStreamVideoTrack for downstream consumption.
2459
+ * Base class for real-time video filters.
2460
+ *
2461
+ * It sets up the full pipeline that reads frames from the input track,
2462
+ * processes them, and outputs a new track with your effect applied. Subclasses
2463
+ * only need to implement `initialize` (run once before processing starts) and
2464
+ * `transform` (called for every frame).
2465
+ *
2466
+ * Everything else—canvas setup, performance tracking, error handling, and
2467
+ * clean shutdown is handled for you. Calling `start()` returns a processed
2468
+ * `MediaStreamTrack` ready to use.
2468
2469
  */
2469
- class VirtualBackground {
2470
- constructor(track, options = {}, hooks = {}) {
2470
+ class BaseVideoProcessor {
2471
+ /**
2472
+ * Constructs a new instance.
2473
+ */
2474
+ constructor(track, hooks = {}) {
2471
2475
  this.track = track;
2472
- this.options = options;
2473
- this.hooks = hooks;
2474
- this.segmenter = null;
2475
- this.isSegmenterReady = false;
2476
- this.segmenterDelayTotal = 0;
2476
+ this.abortController = new AbortController();
2477
2477
  this.frames = 0;
2478
+ this.delayTotal = 0;
2478
2479
  this.lastStatsTime = 0;
2479
2480
  this.processor = new TrackProcessor({ track });
2480
2481
  this.generator = new TrackGenerator({
2481
2482
  kind: 'video',
2482
2483
  signalTarget: track,
2483
2484
  });
2484
- this.abortController = new AbortController();
2485
+ this.hooks = hooks;
2485
2486
  }
2486
2487
  async start() {
2487
- const { onError } = this.hooks;
2488
2488
  const { readable } = this.processor;
2489
2489
  const { writable } = this.generator;
2490
- const displayWidth = this.track.getSettings().width ?? 1280;
2491
- const displayHeight = this.track.getSettings().height ?? 720;
2492
- this.canvas = new OffscreenCanvas(displayWidth, displayHeight);
2493
- this.webGlRenderer = new WebGLRenderer(this.canvas);
2494
- await this.initializeSegmenter();
2495
- const opts = await this.initializeSegmenterOptions();
2490
+ const { width = 1280, height = 720 } = this.track.getSettings();
2491
+ this.canvas = new OffscreenCanvas(width, height);
2492
+ await this.initialize();
2496
2493
  const transformStream = new TransformStream({
2497
2494
  transform: async (frame, controller) => {
2498
2495
  try {
2499
- if (this.abortController.signal.aborted) {
2496
+ if (this.abortController.signal.aborted)
2500
2497
  return frame.close();
2498
+ if (this.canvas.width !== frame.displayWidth ||
2499
+ this.canvas.height !== frame.displayHeight) {
2500
+ this.canvas.width = frame.displayWidth;
2501
+ this.canvas.height = frame.displayHeight;
2501
2502
  }
2502
- const processed = await this.transform(frame, opts);
2503
+ const start = performance.now();
2504
+ const processed = await this.transform(frame);
2505
+ const delay = performance.now() - start;
2506
+ this.updateStats(delay);
2503
2507
  controller.enqueue(processed);
2504
2508
  }
2505
2509
  catch (e) {
2506
- console.error('[virtual-background] error processing frame:', e);
2507
2510
  this.hooks.onError?.(e);
2508
- if (!this.abortController.signal.aborted) {
2509
- controller.enqueue(frame);
2510
- }
2511
2511
  }
2512
2512
  finally {
2513
2513
  frame.close();
2514
2514
  }
2515
2515
  },
2516
- flush: () => {
2517
- if (this.segmenter) {
2518
- this.segmenter.close();
2519
- this.segmenter = null;
2520
- }
2521
- this.isSegmenterReady = false;
2522
- },
2516
+ flush: () => this.onFlush(),
2523
2517
  });
2524
- const signal = this.abortController.signal;
2525
2518
  readable
2526
- .pipeThrough(transformStream, { signal })
2527
- .pipeTo(writable)
2519
+ .pipeThrough(transformStream, { signal: this.abortController.signal })
2520
+ .pipeTo(writable, { signal: this.abortController.signal })
2528
2521
  .catch((e) => {
2529
- if (e.name !== 'AbortError') {
2530
- console.error('[virtual-background] Error processing track:', e);
2531
- onError?.(e);
2522
+ if (e.name !== 'AbortError' && e.name !== 'InvalidStateError') {
2523
+ console.error(`[${this.processorName}] Error processing track:`, e);
2524
+ this.hooks.onError?.(e);
2532
2525
  }
2533
2526
  });
2534
2527
  return this.generator;
2535
2528
  }
2536
- /**
2537
- * Loads and initializes the MediaPipe `ImageSegmenter`.
2538
- */
2529
+ stop() {
2530
+ this.abortController.abort();
2531
+ this.generator.stop();
2532
+ this.onStop();
2533
+ }
2534
+ updateStats(delay) {
2535
+ this.frames++;
2536
+ this.delayTotal += delay;
2537
+ const now = performance.now();
2538
+ if (this.lastStatsTime === 0) {
2539
+ this.lastStatsTime = now;
2540
+ return;
2541
+ }
2542
+ if (now - this.lastStatsTime >= 1000) {
2543
+ const avgDelay = Math.round((this.delayTotal / this.frames) * 100) / 100;
2544
+ const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
2545
+ this.hooks.onStats?.({ delay: avgDelay, fps, timestamp: now });
2546
+ this.frames = 0;
2547
+ this.delayTotal = 0;
2548
+ this.lastStatsTime = now;
2549
+ }
2550
+ }
2551
+ onFlush() { }
2552
+ onStop() { }
2553
+ get processorName() {
2554
+ return 'base-processor';
2555
+ }
2556
+ }
2557
+
2558
+ /**
2559
+ * Wraps a video MediaStreamTrack in a real-time processing pipeline.
2560
+ * Incoming frames are processed through a transformer and re-emitted
2561
+ * on a new MediaStreamVideoTrack for downstream consumption.
2562
+ */
2563
+ class VirtualBackground extends BaseVideoProcessor {
2564
+ constructor(track, options = {}, hooks = {}) {
2565
+ super(track, hooks);
2566
+ this.options = options;
2567
+ this.segmenter = null;
2568
+ this.isSegmenterReady = false;
2569
+ this.latestCategoryMask = undefined;
2570
+ this.latestConfidenceMask = undefined;
2571
+ this.lastFrameTime = -1;
2572
+ this.count = 0;
2573
+ }
2574
+ async initialize() {
2575
+ this.webGlRenderer = new WebGLRenderer(this.canvas);
2576
+ await this.initializeSegmenter();
2577
+ }
2539
2578
  async initializeSegmenter() {
2540
2579
  try {
2541
- const basePath = this.options?.basePath ||
2580
+ this.opts = await this.initializeSegmenterOptions();
2581
+ const basePath = this.options.basePath ||
2542
2582
  `https://unpkg.com/${packageName}@${version}/mediapipe`;
2543
- const defaultModelPath = `${basePath}/models/selfie_segmenter.tflite`;
2544
- const model = this.options?.modelPath || defaultModelPath;
2545
- const wasmPath = `${basePath}/wasm`;
2546
- const fileset = await tasksVision.FilesetResolver.forVisionTasks(wasmPath);
2583
+ const model = this.options.modelPath || `${basePath}/models/selfie_segmenter.tflite`;
2584
+ const fileset = await tasksVision.FilesetResolver.forVisionTasks(`${basePath}/wasm`);
2547
2585
  this.segmenter = await tasksVision.ImageSegmenter.createFromOptions(fileset, {
2548
- baseOptions: {
2549
- modelAssetPath: model,
2550
- delegate: 'GPU',
2551
- },
2586
+ baseOptions: { modelAssetPath: model, delegate: 'GPU' },
2552
2587
  runningMode: 'VIDEO',
2553
2588
  outputCategoryMask: true,
2554
2589
  outputConfidenceMasks: true,
@@ -2557,75 +2592,50 @@ class VirtualBackground {
2557
2592
  this.isSegmenterReady = true;
2558
2593
  }
2559
2594
  catch (error) {
2560
- console.error('[virtual-background] Failed to initialize MediaPipe segmenter:', error);
2595
+ console.error('[virtual-background] Segmenter init failed:', error);
2561
2596
  this.isSegmenterReady = false;
2562
2597
  }
2563
2598
  }
2564
- /**
2565
- * Processes a single video frame.
2566
- *
2567
- * Performs segmentation via MediaPipe and then composites the frame
2568
- * through the WebGL renderer to apply background effects.
2569
- *
2570
- * @param frame - The incoming frame from the processor.
2571
- * @param opts - The segmentation options to use.
2572
- *
2573
- * @returns A new `VideoFrame` containing the processed image.
2574
- */
2575
- async transform(frame, opts) {
2576
- if (this.isSegmenterReady && this.segmenter) {
2577
- try {
2578
- const start = performance.now();
2579
- await new Promise((resolve) => {
2580
- this.segmenter.segmentForVideo(frame, frame.timestamp, (result) => {
2581
- const categoryMask = result.categoryMask.getAsWebGLTexture();
2582
- const confidenceMask = result.confidenceMasks[0].getAsWebGLTexture();
2583
- this.webGlRenderer.render(frame, opts, categoryMask, confidenceMask);
2584
- const now = performance.now();
2585
- this.segmenterDelayTotal += now - start;
2586
- this.frames++;
2587
- if (this.lastStatsTime === 0) {
2588
- this.lastStatsTime = now;
2589
- }
2590
- if (now - this.lastStatsTime > 1000) {
2591
- const delay = Math.round((this.segmenterDelayTotal / this.frames) * 100) /
2592
- 100;
2593
- const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
2594
- this.hooks.onStats?.({ delay, fps, timestamp: now });
2595
- this.lastStatsTime = now;
2596
- this.segmenterDelayTotal = 0;
2597
- this.frames = 0;
2598
- }
2599
- resolve();
2600
- });
2601
- });
2602
- }
2603
- catch (error) {
2604
- console.error('[virtual-background] Error during segmentation:', error);
2605
- }
2599
+ async transform(frame) {
2600
+ const currentTime = frame.timestamp;
2601
+ const hasNewFrame = currentTime !== this.lastFrameTime;
2602
+ this.lastFrameTime = currentTime;
2603
+ if (hasNewFrame && this.isSegmenterReady && this.segmenter) {
2604
+ await this.runSegmentation(frame);
2605
+ this.webGlRenderer.render(frame, this.opts, this.latestCategoryMask, this.latestConfidenceMask);
2606
2606
  }
2607
2607
  return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
2608
2608
  }
2609
- async loadBackground(url) {
2610
- if (!url) {
2611
- return;
2612
- }
2613
- const response = await fetch(url);
2614
- if (!response.ok) {
2615
- console.error(`[virtual-background] Failed to fetch background source ${url} (status: ${response.status})`);
2609
+ async runSegmentation(frame) {
2610
+ if (!this.segmenter)
2616
2611
  return;
2617
- }
2618
- const blob = await response.blob();
2619
- const imageBitmap = await createImageBitmap(blob);
2620
- return { type: 'image', media: imageBitmap, url };
2612
+ return new Promise((resolve) => {
2613
+ const timestamp = Math.floor(performance.now());
2614
+ this.segmenter.segmentForVideo(frame, timestamp, (result) => {
2615
+ try {
2616
+ this.latestCategoryMask = result.categoryMask?.getAsWebGLTexture();
2617
+ this.latestConfidenceMask =
2618
+ result.confidenceMasks?.[0]?.getAsWebGLTexture();
2619
+ }
2620
+ catch (err) {
2621
+ console.error('[virtual-background] segmentation error:', err);
2622
+ this.hooks.onError?.(err);
2623
+ }
2624
+ finally {
2625
+ result.close();
2626
+ resolve();
2627
+ }
2628
+ });
2629
+ });
2621
2630
  }
2622
2631
  async initializeSegmenterOptions() {
2623
2632
  const isSelfieMode = this.options.modelPath
2624
- ? this.options.modelPath?.includes('selfie_segmenter')
2633
+ ? this.options.modelPath.includes('selfie_segmenter')
2625
2634
  : true;
2626
2635
  if (this.options.backgroundFilter === 'image') {
2636
+ const source = await this.loadBackground(this.options.backgroundImage);
2627
2637
  return {
2628
- backgroundSource: await this.loadBackground(this.options.backgroundImage),
2638
+ backgroundSource: source,
2629
2639
  bgBlur: 0,
2630
2640
  bgBlurRadius: 0,
2631
2641
  isSelfieMode,
@@ -2640,28 +2650,340 @@ class VirtualBackground {
2640
2650
  };
2641
2651
  }
2642
2652
  const numeric = blurLevel ?? 5;
2643
- const bgBlur = Math.min(numeric * 3, 30);
2644
- const bgBlurRadius = Math.min(numeric, 10);
2645
2653
  return {
2646
2654
  backgroundSource: undefined,
2647
- bgBlur,
2648
- bgBlurRadius,
2655
+ bgBlur: Math.min(numeric * 3, 30),
2656
+ bgBlurRadius: Math.min(numeric, 10),
2649
2657
  isSelfieMode,
2650
2658
  };
2651
2659
  }
2652
- stop() {
2653
- this.abortController.abort();
2654
- this.webGlRenderer.close();
2655
- this.generator.stop();
2656
- if (this.segmenter) {
2657
- this.segmenter.close();
2658
- this.segmenter = null;
2659
- }
2660
+ async loadBackground(url) {
2661
+ if (!url)
2662
+ return null;
2663
+ const result = await fetch(url);
2664
+ if (!result.ok)
2665
+ return null;
2666
+ return {
2667
+ type: 'image',
2668
+ media: await createImageBitmap(await result.blob()),
2669
+ url,
2670
+ };
2671
+ }
2672
+ onFlush() {
2673
+ this.destroySegmenter();
2674
+ }
2675
+ onStop() {
2676
+ this.webGlRenderer?.close();
2677
+ this.destroySegmenter();
2678
+ }
2679
+ destroySegmenter() {
2680
+ this.segmenter?.close();
2681
+ this.segmenter = null;
2660
2682
  this.isSegmenterReady = false;
2661
2683
  }
2684
+ get processorName() {
2685
+ return 'background-processor';
2686
+ }
2687
+ }
2688
+
2689
+ /**
2690
+ * Simple WebGL renderer for full-screen Gaussian blur.
2691
+ * Uses a two-pass separable Gaussian blur (horizontal then vertical).
2692
+ * Optimized for moderation use cases by blurring at reduced resolution (15% scale)
2693
+ * and upscaling back to full resolution for output.
2694
+ */
2695
+ class FullScreenBlurRenderer {
2696
+ constructor(canvas) {
2697
+ this.inputTexture = null;
2698
+ this.isRunning = false;
2699
+ this.targetWidth = 0;
2700
+ this.targetHeight = 0;
2701
+ this.weightCache = new Map();
2702
+ this.canvas = canvas;
2703
+ const gl = canvas.getContext('webgl2', {
2704
+ alpha: false,
2705
+ antialias: false,
2706
+ desynchronized: true,
2707
+ });
2708
+ if (!gl)
2709
+ throw new Error('WebGL2 not supported');
2710
+ this.gl = gl;
2711
+ const vertexShaderSource = `#version 300 es
2712
+ precision highp float;
2713
+ in vec2 a_position;
2714
+ in vec2 a_texCoord;
2715
+ out vec2 v_texCoord;
2716
+ void main() {
2717
+ v_texCoord = a_texCoord;
2718
+ gl_Position = vec4(a_position, 0.0, 1.0);
2719
+ }
2720
+ `;
2721
+ const fragmentShaderSource = `#version 300 es
2722
+ precision highp float;
2723
+ in vec2 v_texCoord;
2724
+ out vec4 outColor;
2725
+ uniform sampler2D u_image;
2726
+ uniform vec2 u_texelSize;
2727
+ uniform vec2 u_direction;
2728
+ uniform float u_weights[25];
2729
+ void main() {
2730
+ vec4 color = vec4(0.0);
2731
+ for (int i = -12; i <= 12; i++) {
2732
+ float w = u_weights[i + 12];
2733
+ if (w == 0.0) continue;
2734
+ vec2 offset = float(i) * u_direction * u_texelSize;
2735
+ color += w * texture(u_image, v_texCoord + offset);
2736
+ }
2737
+ outColor = color;
2738
+ }
2739
+ `;
2740
+ this.blurProgramHandle = this.createAndLinkProgram(vertexShaderSource, fragmentShaderSource);
2741
+ const passthroughFragmentShaderSource = `#version 300 es
2742
+ precision highp float;
2743
+ in vec2 v_texCoord;
2744
+ out vec4 outColor;
2745
+ uniform sampler2D u_image;
2746
+ void main() {
2747
+ outColor = texture(u_image, v_texCoord);
2748
+ }
2749
+ `;
2750
+ this.passthroughProgramHandle = this.createAndLinkProgram(vertexShaderSource, passthroughFragmentShaderSource);
2751
+ const blurProgram = this.blurProgramHandle;
2752
+ const passthroughProgram = this.passthroughProgramHandle;
2753
+ this.blurLocations = {
2754
+ positionLocation: gl.getAttribLocation(blurProgram, 'a_position'),
2755
+ texCoordLocation: gl.getAttribLocation(blurProgram, 'a_texCoord'),
2756
+ imageLocation: gl.getUniformLocation(blurProgram, 'u_image'),
2757
+ texelSizeLocation: gl.getUniformLocation(blurProgram, 'u_texelSize'),
2758
+ directionLocation: gl.getUniformLocation(blurProgram, 'u_direction'),
2759
+ weightsLocation: gl.getUniformLocation(blurProgram, 'u_weights'),
2760
+ };
2761
+ this.passthroughLocations = {
2762
+ positionLocation: gl.getAttribLocation(passthroughProgram, 'a_position'),
2763
+ texCoordLocation: gl.getAttribLocation(passthroughProgram, 'a_texCoord'),
2764
+ imageLocation: gl.getUniformLocation(passthroughProgram, 'u_image'),
2765
+ };
2766
+ this.positionBuffer = gl.createBuffer();
2767
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2768
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
2769
+ this.texCoordBuffer = gl.createBuffer();
2770
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2771
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
2772
+ const createTexture2D = () => {
2773
+ const tex = gl.createTexture();
2774
+ if (!tex)
2775
+ throw new Error('Failed to create texture');
2776
+ gl.bindTexture(gl.TEXTURE_2D, tex);
2777
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2778
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2779
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
2780
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
2781
+ gl.bindTexture(gl.TEXTURE_2D, null);
2782
+ return tex;
2783
+ };
2784
+ const createFramebufferForTexture = (tex) => {
2785
+ const fb = gl.createFramebuffer();
2786
+ if (!fb)
2787
+ throw new Error('Failed to create framebuffer');
2788
+ gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
2789
+ gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
2790
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2791
+ return fb;
2792
+ };
2793
+ this.pingTexture = createTexture2D();
2794
+ this.pongTexture = createTexture2D();
2795
+ this.pingFbo = createFramebufferForTexture(this.pingTexture);
2796
+ this.pongFbo = createFramebufferForTexture(this.pongTexture);
2797
+ this.inputTexture = createTexture2D();
2798
+ gl.useProgram(blurProgram);
2799
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2800
+ gl.enableVertexAttribArray(this.blurLocations.positionLocation);
2801
+ gl.vertexAttribPointer(this.blurLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
2802
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2803
+ gl.enableVertexAttribArray(this.blurLocations.texCoordLocation);
2804
+ gl.vertexAttribPointer(this.blurLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
2805
+ if (this.blurLocations.imageLocation) {
2806
+ gl.uniform1i(this.blurLocations.imageLocation, 0);
2807
+ }
2808
+ this.isRunning = true;
2809
+ }
2810
+ createAndLinkProgram(vsSource, fsSource) {
2811
+ const gl = this.gl;
2812
+ const vs = this.createShader(gl.VERTEX_SHADER, vsSource);
2813
+ const fs = this.createShader(gl.FRAGMENT_SHADER, fsSource);
2814
+ const prog = gl.createProgram();
2815
+ if (!prog)
2816
+ throw new Error('Failed to create program');
2817
+ gl.attachShader(prog, vs);
2818
+ gl.attachShader(prog, fs);
2819
+ gl.linkProgram(prog);
2820
+ if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
2821
+ throw new Error('Shader link failed: ' + gl.getProgramInfoLog(prog));
2822
+ }
2823
+ gl.deleteShader(vs);
2824
+ gl.deleteShader(fs);
2825
+ return prog;
2826
+ }
2827
+ createShader(type, source) {
2828
+ const gl = this.gl;
2829
+ const shader = gl.createShader(type);
2830
+ if (!shader)
2831
+ throw new Error('Failed to create shader');
2832
+ gl.shaderSource(shader, source);
2833
+ gl.compileShader(shader);
2834
+ if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
2835
+ throw new Error('Shader compile failed: ' + gl.getShaderInfoLog(shader));
2836
+ }
2837
+ return shader;
2838
+ }
2839
+ getGaussianWeights(radius) {
2840
+ const r = Math.max(0, Math.min(radius | 0, 12));
2841
+ const cached = this.weightCache.get(r);
2842
+ if (cached)
2843
+ return cached;
2844
+ const weights = new Float32Array(25);
2845
+ if (r === 0) {
2846
+ weights[12] = 1.0;
2847
+ this.weightCache.set(r, weights);
2848
+ return weights;
2849
+ }
2850
+ const sigma = r * 0.6;
2851
+ let sum = 0;
2852
+ for (let i = -r; i <= r; i++) {
2853
+ const w = Math.exp(-(i * i) / (2 * sigma * sigma));
2854
+ weights[i + 12] = w;
2855
+ sum += w;
2856
+ }
2857
+ for (let i = -r; i <= r; i++) {
2858
+ weights[i + 12] /= sum;
2859
+ }
2860
+ this.weightCache.set(r, weights);
2861
+ return weights;
2862
+ }
2863
+ render(frame, radius) {
2864
+ if (!this.isRunning)
2865
+ return;
2866
+ const gl = this.gl;
2867
+ const width = frame.displayWidth;
2868
+ const height = frame.displayHeight;
2869
+ if (!width || !height)
2870
+ return;
2871
+ if (this.canvas.width !== width || this.canvas.height !== height) {
2872
+ this.canvas.width = width;
2873
+ this.canvas.height = height;
2874
+ }
2875
+ const scale = 0.15;
2876
+ const scaledWidth = Math.max(1, Math.floor(width * scale));
2877
+ const scaledHeight = Math.max(1, Math.floor(height * scale));
2878
+ if (scaledWidth !== this.targetWidth ||
2879
+ scaledHeight !== this.targetHeight) {
2880
+ this.targetWidth = scaledWidth;
2881
+ this.targetHeight = scaledHeight;
2882
+ gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
2883
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2884
+ gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
2885
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2886
+ gl.bindTexture(gl.TEXTURE_2D, null);
2887
+ }
2888
+ gl.activeTexture(gl.TEXTURE0);
2889
+ gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
2890
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
2891
+ gl.useProgram(this.blurProgramHandle);
2892
+ if (this.blurLocations.texelSizeLocation) {
2893
+ gl.uniform2f(this.blurLocations.texelSizeLocation, 1.0 / scaledWidth, 1.0 / scaledHeight);
2894
+ }
2895
+ const weights = this.getGaussianWeights(radius);
2896
+ if (this.blurLocations.weightsLocation) {
2897
+ gl.uniform1fv(this.blurLocations.weightsLocation, weights);
2898
+ }
2899
+ gl.viewport(0, 0, scaledWidth, scaledHeight);
2900
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this.pingFbo);
2901
+ gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
2902
+ if (this.blurLocations.directionLocation) {
2903
+ gl.uniform2f(this.blurLocations.directionLocation, 1.0, 0.0);
2904
+ }
2905
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
2906
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this.pongFbo);
2907
+ gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
2908
+ if (this.blurLocations.directionLocation) {
2909
+ gl.uniform2f(this.blurLocations.directionLocation, 0.0, 1.0);
2910
+ }
2911
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
2912
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2913
+ gl.viewport(0, 0, width, height);
2914
+ gl.useProgram(this.passthroughProgramHandle);
2915
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2916
+ gl.enableVertexAttribArray(this.passthroughLocations.positionLocation);
2917
+ gl.vertexAttribPointer(this.passthroughLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
2918
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2919
+ gl.enableVertexAttribArray(this.passthroughLocations.texCoordLocation);
2920
+ gl.vertexAttribPointer(this.passthroughLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
2921
+ gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
2922
+ if (this.passthroughLocations.imageLocation) {
2923
+ gl.uniform1i(this.passthroughLocations.imageLocation, 0);
2924
+ }
2925
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
2926
+ }
2927
+ close() {
2928
+ if (!this.isRunning)
2929
+ return;
2930
+ this.isRunning = false;
2931
+ const gl = this.gl;
2932
+ if (this.pingFbo)
2933
+ gl.deleteFramebuffer(this.pingFbo);
2934
+ if (this.pongFbo)
2935
+ gl.deleteFramebuffer(this.pongFbo);
2936
+ if (this.pingTexture)
2937
+ gl.deleteTexture(this.pingTexture);
2938
+ if (this.pongTexture)
2939
+ gl.deleteTexture(this.pongTexture);
2940
+ if (this.inputTexture)
2941
+ gl.deleteTexture(this.inputTexture);
2942
+ if (this.positionBuffer)
2943
+ gl.deleteBuffer(this.positionBuffer);
2944
+ if (this.texCoordBuffer)
2945
+ gl.deleteBuffer(this.texCoordBuffer);
2946
+ gl.deleteProgram(this.blurProgramHandle);
2947
+ gl.deleteProgram(this.passthroughProgramHandle);
2948
+ }
2949
+ }
2950
+
2951
+ /**
2952
+ * A video filter that applies a full-screen blur to each frame.
2953
+ *
2954
+ * It uses a WebGL renderer to blur the incoming camera track and outputs
2955
+ * a new track with the effect applied. Setup and frame handling are managed
2956
+ * by the base processor.
2957
+ */
2958
+ class FullScreenBlur extends BaseVideoProcessor {
2959
+ /**
2960
+ * Creates a new full-screen blur processor for the given video track.
2961
+ *
2962
+ * @param track - The input camera track to blur.
2963
+ * @param options - Optional settings such as the blur radius.
2964
+ * @param hooks - Optional callbacks for stats and error reporting.
2965
+ */
2966
+ constructor(track, options = {}, hooks = {}) {
2967
+ super(track, hooks);
2968
+ this.blurRadius = options.blurRadius ?? 6;
2969
+ }
2970
+ async initialize() {
2971
+ this.blurRenderer = new FullScreenBlurRenderer(this.canvas);
2972
+ }
2973
+ async transform(frame) {
2974
+ this.blurRenderer.render(frame, this.blurRadius);
2975
+ return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
2976
+ }
2977
+ onStop() {
2978
+ this.blurRenderer?.close();
2979
+ }
2980
+ get processorName() {
2981
+ return 'fullscreen-blur';
2982
+ }
2662
2983
  }
2663
2984
 
2664
2985
  exports.BACKGROUND_BLUR_MAP = BACKGROUND_BLUR_MAP;
2986
+ exports.FullScreenBlur = FullScreenBlur;
2665
2987
  exports.VirtualBackground = VirtualBackground;
2666
2988
  exports.createRenderer = createRenderer;
2667
2989
  exports.isMediaPipePlatformSupported = isMediaPipePlatformSupported;