@stream-io/video-filters-web 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -5,3 +5,4 @@ export * from './src/legacy/tflite';
5
5
  export * from './src/mediapipe';
6
6
  export * from './src/types';
7
7
  export * from './src/VirtualBackground';
8
+ export * from './src/FullScreenBlur';
package/dist/index.es.js CHANGED
@@ -1559,7 +1559,7 @@ const createTFLiteSIMDModule = (__Module) => {
1559
1559
  return __Module.ready;
1560
1560
  };
1561
1561
 
1562
- const version = "0.5.0";
1562
+ const version = "0.6.0";
1563
1563
  const packageName = "@stream-io/video-filters-web";
1564
1564
 
1565
1565
  // @ts-expect-error - module is not declared
@@ -2403,7 +2403,6 @@ class FallbackProcessor {
2403
2403
  let timestamp = 0;
2404
2404
  const frameRate = track.getSettings().frameRate || 30;
2405
2405
  let frameDuration = 1000 / frameRate;
2406
- let lastVideoTime = -1;
2407
2406
  this.workerTimer = new WorkerTimer({ useWorker: true });
2408
2407
  this.readable = new ReadableStream({
2409
2408
  start: async () => {
@@ -2425,13 +2424,6 @@ class FallbackProcessor {
2425
2424
  await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration - delta));
2426
2425
  }
2427
2426
  timestamp = performance.now();
2428
- const currentTime = this.video.currentTime;
2429
- const hasNewFrame = currentTime !== lastVideoTime;
2430
- if (!hasNewFrame) {
2431
- await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration));
2432
- return;
2433
- }
2434
- lastVideoTime = currentTime;
2435
2427
  if (canvas.width !== this.video.videoWidth ||
2436
2428
  canvas.height !== this.video.videoHeight) {
2437
2429
  canvas.width = this.video.videoWidth;
@@ -2439,7 +2431,9 @@ class FallbackProcessor {
2439
2431
  }
2440
2432
  ctx.drawImage(this.video, 0, 0);
2441
2433
  try {
2442
- const frame = new VideoFrame(canvas, { timestamp });
2434
+ const frame = new VideoFrame(canvas, {
2435
+ timestamp: Math.round(this.video.currentTime * 1000000),
2436
+ });
2443
2437
  controller.enqueue(frame);
2444
2438
  }
2445
2439
  catch (err) {
@@ -2460,93 +2454,134 @@ const TrackProcessor = typeof MediaStreamTrackProcessor !== 'undefined'
2460
2454
  : FallbackProcessor;
2461
2455
 
2462
2456
  /**
2463
- * Wraps a video MediaStreamTrack in a real-time processing pipeline.
2464
- * Incoming frames are processed through a transformer and re-emitted
2465
- * on a new MediaStreamVideoTrack for downstream consumption.
2457
+ * Base class for real-time video filters.
2458
+ *
2459
+ * It sets up the full pipeline that reads frames from the input track,
2460
+ * processes them, and outputs a new track with your effect applied. Subclasses
2461
+ * only need to implement `initialize` (run once before processing starts) and
2462
+ * `transform` (called for every frame).
2463
+ *
2464
+ * Everything else—canvas setup, performance tracking, error handling, and
2465
+ * clean shutdown is handled for you. Calling `start()` returns a processed
2466
+ * `MediaStreamTrack` ready to use.
2466
2467
  */
2467
- class VirtualBackground {
2468
- constructor(track, options = {}, hooks = {}) {
2468
+ class BaseVideoProcessor {
2469
+ /**
2470
+ * Constructs a new instance.
2471
+ */
2472
+ constructor(track, hooks = {}) {
2469
2473
  this.track = track;
2470
- this.options = options;
2471
- this.hooks = hooks;
2472
- this.segmenter = null;
2473
- this.isSegmenterReady = false;
2474
- this.segmenterDelayTotal = 0;
2474
+ this.abortController = new AbortController();
2475
2475
  this.frames = 0;
2476
+ this.delayTotal = 0;
2476
2477
  this.lastStatsTime = 0;
2477
2478
  this.processor = new TrackProcessor({ track });
2478
2479
  this.generator = new TrackGenerator({
2479
2480
  kind: 'video',
2480
2481
  signalTarget: track,
2481
2482
  });
2482
- this.abortController = new AbortController();
2483
+ this.hooks = hooks;
2483
2484
  }
2484
2485
  async start() {
2485
- const { onError } = this.hooks;
2486
2486
  const { readable } = this.processor;
2487
2487
  const { writable } = this.generator;
2488
- const displayWidth = this.track.getSettings().width ?? 1280;
2489
- const displayHeight = this.track.getSettings().height ?? 720;
2490
- this.canvas = new OffscreenCanvas(displayWidth, displayHeight);
2491
- this.webGlRenderer = new WebGLRenderer(this.canvas);
2492
- await this.initializeSegmenter();
2493
- const opts = await this.initializeSegmenterOptions();
2488
+ const { width = 1280, height = 720 } = this.track.getSettings();
2489
+ this.canvas = new OffscreenCanvas(width, height);
2490
+ await this.initialize();
2494
2491
  const transformStream = new TransformStream({
2495
2492
  transform: async (frame, controller) => {
2496
2493
  try {
2497
- if (this.abortController.signal.aborted) {
2494
+ if (this.abortController.signal.aborted)
2498
2495
  return frame.close();
2496
+ if (this.canvas.width !== frame.displayWidth ||
2497
+ this.canvas.height !== frame.displayHeight) {
2498
+ this.canvas.width = frame.displayWidth;
2499
+ this.canvas.height = frame.displayHeight;
2499
2500
  }
2500
- const processed = await this.transform(frame, opts);
2501
+ const start = performance.now();
2502
+ const processed = await this.transform(frame);
2503
+ const delay = performance.now() - start;
2504
+ this.updateStats(delay);
2501
2505
  controller.enqueue(processed);
2502
2506
  }
2503
2507
  catch (e) {
2504
- console.error('[virtual-background] error processing frame:', e);
2505
2508
  this.hooks.onError?.(e);
2506
- if (!this.abortController.signal.aborted) {
2507
- controller.enqueue(frame);
2508
- }
2509
2509
  }
2510
2510
  finally {
2511
2511
  frame.close();
2512
2512
  }
2513
2513
  },
2514
- flush: () => {
2515
- if (this.segmenter) {
2516
- this.segmenter.close();
2517
- this.segmenter = null;
2518
- }
2519
- this.isSegmenterReady = false;
2520
- },
2514
+ flush: () => this.onFlush(),
2521
2515
  });
2522
- const signal = this.abortController.signal;
2523
2516
  readable
2524
- .pipeThrough(transformStream, { signal })
2525
- .pipeTo(writable)
2517
+ .pipeThrough(transformStream, { signal: this.abortController.signal })
2518
+ .pipeTo(writable, { signal: this.abortController.signal })
2526
2519
  .catch((e) => {
2527
- if (e.name !== 'AbortError') {
2528
- console.error('[virtual-background] Error processing track:', e);
2529
- onError?.(e);
2520
+ if (e.name !== 'AbortError' && e.name !== 'InvalidStateError') {
2521
+ console.error(`[${this.processorName}] Error processing track:`, e);
2522
+ this.hooks.onError?.(e);
2530
2523
  }
2531
2524
  });
2532
2525
  return this.generator;
2533
2526
  }
2534
- /**
2535
- * Loads and initializes the MediaPipe `ImageSegmenter`.
2536
- */
2527
+ stop() {
2528
+ this.abortController.abort();
2529
+ this.generator.stop();
2530
+ this.onStop();
2531
+ }
2532
+ updateStats(delay) {
2533
+ this.frames++;
2534
+ this.delayTotal += delay;
2535
+ const now = performance.now();
2536
+ if (this.lastStatsTime === 0) {
2537
+ this.lastStatsTime = now;
2538
+ return;
2539
+ }
2540
+ if (now - this.lastStatsTime >= 1000) {
2541
+ const avgDelay = Math.round((this.delayTotal / this.frames) * 100) / 100;
2542
+ const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
2543
+ this.hooks.onStats?.({ delay: avgDelay, fps, timestamp: now });
2544
+ this.frames = 0;
2545
+ this.delayTotal = 0;
2546
+ this.lastStatsTime = now;
2547
+ }
2548
+ }
2549
+ onFlush() { }
2550
+ onStop() { }
2551
+ get processorName() {
2552
+ return 'base-processor';
2553
+ }
2554
+ }
2555
+
2556
+ /**
2557
+ * Wraps a video MediaStreamTrack in a real-time processing pipeline.
2558
+ * Incoming frames are processed through a transformer and re-emitted
2559
+ * on a new MediaStreamVideoTrack for downstream consumption.
2560
+ */
2561
+ class VirtualBackground extends BaseVideoProcessor {
2562
+ constructor(track, options = {}, hooks = {}) {
2563
+ super(track, hooks);
2564
+ this.options = options;
2565
+ this.segmenter = null;
2566
+ this.isSegmenterReady = false;
2567
+ this.latestCategoryMask = undefined;
2568
+ this.latestConfidenceMask = undefined;
2569
+ this.lastFrameTime = -1;
2570
+ this.count = 0;
2571
+ }
2572
+ async initialize() {
2573
+ this.webGlRenderer = new WebGLRenderer(this.canvas);
2574
+ await this.initializeSegmenter();
2575
+ }
2537
2576
  async initializeSegmenter() {
2538
2577
  try {
2539
- const basePath = this.options?.basePath ||
2578
+ this.opts = await this.initializeSegmenterOptions();
2579
+ const basePath = this.options.basePath ||
2540
2580
  `https://unpkg.com/${packageName}@${version}/mediapipe`;
2541
- const defaultModelPath = `${basePath}/models/selfie_segmenter.tflite`;
2542
- const model = this.options?.modelPath || defaultModelPath;
2543
- const wasmPath = `${basePath}/wasm`;
2544
- const fileset = await FilesetResolver.forVisionTasks(wasmPath);
2581
+ const model = this.options.modelPath || `${basePath}/models/selfie_segmenter.tflite`;
2582
+ const fileset = await FilesetResolver.forVisionTasks(`${basePath}/wasm`);
2545
2583
  this.segmenter = await ImageSegmenter.createFromOptions(fileset, {
2546
- baseOptions: {
2547
- modelAssetPath: model,
2548
- delegate: 'GPU',
2549
- },
2584
+ baseOptions: { modelAssetPath: model, delegate: 'GPU' },
2550
2585
  runningMode: 'VIDEO',
2551
2586
  outputCategoryMask: true,
2552
2587
  outputConfidenceMasks: true,
@@ -2555,75 +2590,50 @@ class VirtualBackground {
2555
2590
  this.isSegmenterReady = true;
2556
2591
  }
2557
2592
  catch (error) {
2558
- console.error('[virtual-background] Failed to initialize MediaPipe segmenter:', error);
2593
+ console.error('[virtual-background] Segmenter init failed:', error);
2559
2594
  this.isSegmenterReady = false;
2560
2595
  }
2561
2596
  }
2562
- /**
2563
- * Processes a single video frame.
2564
- *
2565
- * Performs segmentation via MediaPipe and then composites the frame
2566
- * through the WebGL renderer to apply background effects.
2567
- *
2568
- * @param frame - The incoming frame from the processor.
2569
- * @param opts - The segmentation options to use.
2570
- *
2571
- * @returns A new `VideoFrame` containing the processed image.
2572
- */
2573
- async transform(frame, opts) {
2574
- if (this.isSegmenterReady && this.segmenter) {
2575
- try {
2576
- const start = performance.now();
2577
- await new Promise((resolve) => {
2578
- this.segmenter.segmentForVideo(frame, frame.timestamp, (result) => {
2579
- const categoryMask = result.categoryMask.getAsWebGLTexture();
2580
- const confidenceMask = result.confidenceMasks[0].getAsWebGLTexture();
2581
- this.webGlRenderer.render(frame, opts, categoryMask, confidenceMask);
2582
- const now = performance.now();
2583
- this.segmenterDelayTotal += now - start;
2584
- this.frames++;
2585
- if (this.lastStatsTime === 0) {
2586
- this.lastStatsTime = now;
2587
- }
2588
- if (now - this.lastStatsTime > 1000) {
2589
- const delay = Math.round((this.segmenterDelayTotal / this.frames) * 100) /
2590
- 100;
2591
- const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
2592
- this.hooks.onStats?.({ delay, fps, timestamp: now });
2593
- this.lastStatsTime = now;
2594
- this.segmenterDelayTotal = 0;
2595
- this.frames = 0;
2596
- }
2597
- resolve();
2598
- });
2599
- });
2600
- }
2601
- catch (error) {
2602
- console.error('[virtual-background] Error during segmentation:', error);
2603
- }
2597
+ async transform(frame) {
2598
+ const currentTime = frame.timestamp;
2599
+ const hasNewFrame = currentTime !== this.lastFrameTime;
2600
+ this.lastFrameTime = currentTime;
2601
+ if (hasNewFrame && this.isSegmenterReady && this.segmenter) {
2602
+ await this.runSegmentation(frame);
2603
+ this.webGlRenderer.render(frame, this.opts, this.latestCategoryMask, this.latestConfidenceMask);
2604
2604
  }
2605
2605
  return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
2606
2606
  }
2607
- async loadBackground(url) {
2608
- if (!url) {
2609
- return;
2610
- }
2611
- const response = await fetch(url);
2612
- if (!response.ok) {
2613
- console.error(`[virtual-background] Failed to fetch background source ${url} (status: ${response.status})`);
2607
+ async runSegmentation(frame) {
2608
+ if (!this.segmenter)
2614
2609
  return;
2615
- }
2616
- const blob = await response.blob();
2617
- const imageBitmap = await createImageBitmap(blob);
2618
- return { type: 'image', media: imageBitmap, url };
2610
+ return new Promise((resolve) => {
2611
+ const timestamp = Math.floor(performance.now());
2612
+ this.segmenter.segmentForVideo(frame, timestamp, (result) => {
2613
+ try {
2614
+ this.latestCategoryMask = result.categoryMask?.getAsWebGLTexture();
2615
+ this.latestConfidenceMask =
2616
+ result.confidenceMasks?.[0]?.getAsWebGLTexture();
2617
+ }
2618
+ catch (err) {
2619
+ console.error('[virtual-background] segmentation error:', err);
2620
+ this.hooks.onError?.(err);
2621
+ }
2622
+ finally {
2623
+ result.close();
2624
+ resolve();
2625
+ }
2626
+ });
2627
+ });
2619
2628
  }
2620
2629
  async initializeSegmenterOptions() {
2621
2630
  const isSelfieMode = this.options.modelPath
2622
- ? this.options.modelPath?.includes('selfie_segmenter')
2631
+ ? this.options.modelPath.includes('selfie_segmenter')
2623
2632
  : true;
2624
2633
  if (this.options.backgroundFilter === 'image') {
2634
+ const source = await this.loadBackground(this.options.backgroundImage);
2625
2635
  return {
2626
- backgroundSource: await this.loadBackground(this.options.backgroundImage),
2636
+ backgroundSource: source,
2627
2637
  bgBlur: 0,
2628
2638
  bgBlurRadius: 0,
2629
2639
  isSelfieMode,
@@ -2638,26 +2648,337 @@ class VirtualBackground {
2638
2648
  };
2639
2649
  }
2640
2650
  const numeric = blurLevel ?? 5;
2641
- const bgBlur = Math.min(numeric * 3, 30);
2642
- const bgBlurRadius = Math.min(numeric, 10);
2643
2651
  return {
2644
2652
  backgroundSource: undefined,
2645
- bgBlur,
2646
- bgBlurRadius,
2653
+ bgBlur: Math.min(numeric * 3, 30),
2654
+ bgBlurRadius: Math.min(numeric, 10),
2647
2655
  isSelfieMode,
2648
2656
  };
2649
2657
  }
2650
- stop() {
2651
- this.abortController.abort();
2652
- this.webGlRenderer.close();
2653
- this.generator.stop();
2654
- if (this.segmenter) {
2655
- this.segmenter.close();
2656
- this.segmenter = null;
2657
- }
2658
+ async loadBackground(url) {
2659
+ if (!url)
2660
+ return null;
2661
+ const result = await fetch(url);
2662
+ if (!result.ok)
2663
+ return null;
2664
+ return {
2665
+ type: 'image',
2666
+ media: await createImageBitmap(await result.blob()),
2667
+ url,
2668
+ };
2669
+ }
2670
+ onFlush() {
2671
+ this.destroySegmenter();
2672
+ }
2673
+ onStop() {
2674
+ this.webGlRenderer?.close();
2675
+ this.destroySegmenter();
2676
+ }
2677
+ destroySegmenter() {
2678
+ this.segmenter?.close();
2679
+ this.segmenter = null;
2658
2680
  this.isSegmenterReady = false;
2659
2681
  }
2682
+ get processorName() {
2683
+ return 'background-processor';
2684
+ }
2685
+ }
2686
+
2687
+ /**
2688
+ * Simple WebGL renderer for full-screen Gaussian blur.
2689
+ * Uses a two-pass separable Gaussian blur (horizontal then vertical).
2690
+ * Optimized for moderation use cases by blurring at reduced resolution (15% scale)
2691
+ * and upscaling back to full resolution for output.
2692
+ */
2693
+ class FullScreenBlurRenderer {
2694
+ constructor(canvas) {
2695
+ this.inputTexture = null;
2696
+ this.isRunning = false;
2697
+ this.targetWidth = 0;
2698
+ this.targetHeight = 0;
2699
+ this.weightCache = new Map();
2700
+ this.canvas = canvas;
2701
+ const gl = canvas.getContext('webgl2', {
2702
+ alpha: false,
2703
+ antialias: false,
2704
+ desynchronized: true,
2705
+ });
2706
+ if (!gl)
2707
+ throw new Error('WebGL2 not supported');
2708
+ this.gl = gl;
2709
+ const vertexShaderSource = `#version 300 es
2710
+ precision highp float;
2711
+ in vec2 a_position;
2712
+ in vec2 a_texCoord;
2713
+ out vec2 v_texCoord;
2714
+ void main() {
2715
+ v_texCoord = a_texCoord;
2716
+ gl_Position = vec4(a_position, 0.0, 1.0);
2717
+ }
2718
+ `;
2719
+ const fragmentShaderSource = `#version 300 es
2720
+ precision highp float;
2721
+ in vec2 v_texCoord;
2722
+ out vec4 outColor;
2723
+ uniform sampler2D u_image;
2724
+ uniform vec2 u_texelSize;
2725
+ uniform vec2 u_direction;
2726
+ uniform float u_weights[25];
2727
+ void main() {
2728
+ vec4 color = vec4(0.0);
2729
+ for (int i = -12; i <= 12; i++) {
2730
+ float w = u_weights[i + 12];
2731
+ if (w == 0.0) continue;
2732
+ vec2 offset = float(i) * u_direction * u_texelSize;
2733
+ color += w * texture(u_image, v_texCoord + offset);
2734
+ }
2735
+ outColor = color;
2736
+ }
2737
+ `;
2738
+ this.blurProgramHandle = this.createAndLinkProgram(vertexShaderSource, fragmentShaderSource);
2739
+ const passthroughFragmentShaderSource = `#version 300 es
2740
+ precision highp float;
2741
+ in vec2 v_texCoord;
2742
+ out vec4 outColor;
2743
+ uniform sampler2D u_image;
2744
+ void main() {
2745
+ outColor = texture(u_image, v_texCoord);
2746
+ }
2747
+ `;
2748
+ this.passthroughProgramHandle = this.createAndLinkProgram(vertexShaderSource, passthroughFragmentShaderSource);
2749
+ const blurProgram = this.blurProgramHandle;
2750
+ const passthroughProgram = this.passthroughProgramHandle;
2751
+ this.blurLocations = {
2752
+ positionLocation: gl.getAttribLocation(blurProgram, 'a_position'),
2753
+ texCoordLocation: gl.getAttribLocation(blurProgram, 'a_texCoord'),
2754
+ imageLocation: gl.getUniformLocation(blurProgram, 'u_image'),
2755
+ texelSizeLocation: gl.getUniformLocation(blurProgram, 'u_texelSize'),
2756
+ directionLocation: gl.getUniformLocation(blurProgram, 'u_direction'),
2757
+ weightsLocation: gl.getUniformLocation(blurProgram, 'u_weights'),
2758
+ };
2759
+ this.passthroughLocations = {
2760
+ positionLocation: gl.getAttribLocation(passthroughProgram, 'a_position'),
2761
+ texCoordLocation: gl.getAttribLocation(passthroughProgram, 'a_texCoord'),
2762
+ imageLocation: gl.getUniformLocation(passthroughProgram, 'u_image'),
2763
+ };
2764
+ this.positionBuffer = gl.createBuffer();
2765
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2766
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
2767
+ this.texCoordBuffer = gl.createBuffer();
2768
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2769
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
2770
+ const createTexture2D = () => {
2771
+ const tex = gl.createTexture();
2772
+ if (!tex)
2773
+ throw new Error('Failed to create texture');
2774
+ gl.bindTexture(gl.TEXTURE_2D, tex);
2775
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
2776
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
2777
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
2778
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
2779
+ gl.bindTexture(gl.TEXTURE_2D, null);
2780
+ return tex;
2781
+ };
2782
+ const createFramebufferForTexture = (tex) => {
2783
+ const fb = gl.createFramebuffer();
2784
+ if (!fb)
2785
+ throw new Error('Failed to create framebuffer');
2786
+ gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
2787
+ gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
2788
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2789
+ return fb;
2790
+ };
2791
+ this.pingTexture = createTexture2D();
2792
+ this.pongTexture = createTexture2D();
2793
+ this.pingFbo = createFramebufferForTexture(this.pingTexture);
2794
+ this.pongFbo = createFramebufferForTexture(this.pongTexture);
2795
+ this.inputTexture = createTexture2D();
2796
+ gl.useProgram(blurProgram);
2797
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2798
+ gl.enableVertexAttribArray(this.blurLocations.positionLocation);
2799
+ gl.vertexAttribPointer(this.blurLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
2800
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2801
+ gl.enableVertexAttribArray(this.blurLocations.texCoordLocation);
2802
+ gl.vertexAttribPointer(this.blurLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
2803
+ if (this.blurLocations.imageLocation) {
2804
+ gl.uniform1i(this.blurLocations.imageLocation, 0);
2805
+ }
2806
+ this.isRunning = true;
2807
+ }
2808
+ createAndLinkProgram(vsSource, fsSource) {
2809
+ const gl = this.gl;
2810
+ const vs = this.createShader(gl.VERTEX_SHADER, vsSource);
2811
+ const fs = this.createShader(gl.FRAGMENT_SHADER, fsSource);
2812
+ const prog = gl.createProgram();
2813
+ if (!prog)
2814
+ throw new Error('Failed to create program');
2815
+ gl.attachShader(prog, vs);
2816
+ gl.attachShader(prog, fs);
2817
+ gl.linkProgram(prog);
2818
+ if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
2819
+ throw new Error('Shader link failed: ' + gl.getProgramInfoLog(prog));
2820
+ }
2821
+ gl.deleteShader(vs);
2822
+ gl.deleteShader(fs);
2823
+ return prog;
2824
+ }
2825
+ createShader(type, source) {
2826
+ const gl = this.gl;
2827
+ const shader = gl.createShader(type);
2828
+ if (!shader)
2829
+ throw new Error('Failed to create shader');
2830
+ gl.shaderSource(shader, source);
2831
+ gl.compileShader(shader);
2832
+ if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
2833
+ throw new Error('Shader compile failed: ' + gl.getShaderInfoLog(shader));
2834
+ }
2835
+ return shader;
2836
+ }
2837
+ getGaussianWeights(radius) {
2838
+ const r = Math.max(0, Math.min(radius | 0, 12));
2839
+ const cached = this.weightCache.get(r);
2840
+ if (cached)
2841
+ return cached;
2842
+ const weights = new Float32Array(25);
2843
+ if (r === 0) {
2844
+ weights[12] = 1.0;
2845
+ this.weightCache.set(r, weights);
2846
+ return weights;
2847
+ }
2848
+ const sigma = r * 0.6;
2849
+ let sum = 0;
2850
+ for (let i = -r; i <= r; i++) {
2851
+ const w = Math.exp(-(i * i) / (2 * sigma * sigma));
2852
+ weights[i + 12] = w;
2853
+ sum += w;
2854
+ }
2855
+ for (let i = -r; i <= r; i++) {
2856
+ weights[i + 12] /= sum;
2857
+ }
2858
+ this.weightCache.set(r, weights);
2859
+ return weights;
2860
+ }
2861
+ render(frame, radius) {
2862
+ if (!this.isRunning)
2863
+ return;
2864
+ const gl = this.gl;
2865
+ const width = frame.displayWidth;
2866
+ const height = frame.displayHeight;
2867
+ if (!width || !height)
2868
+ return;
2869
+ if (this.canvas.width !== width || this.canvas.height !== height) {
2870
+ this.canvas.width = width;
2871
+ this.canvas.height = height;
2872
+ }
2873
+ const scale = 0.15;
2874
+ const scaledWidth = Math.max(1, Math.floor(width * scale));
2875
+ const scaledHeight = Math.max(1, Math.floor(height * scale));
2876
+ if (scaledWidth !== this.targetWidth ||
2877
+ scaledHeight !== this.targetHeight) {
2878
+ this.targetWidth = scaledWidth;
2879
+ this.targetHeight = scaledHeight;
2880
+ gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
2881
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2882
+ gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
2883
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
2884
+ gl.bindTexture(gl.TEXTURE_2D, null);
2885
+ }
2886
+ gl.activeTexture(gl.TEXTURE0);
2887
+ gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
2888
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
2889
+ gl.useProgram(this.blurProgramHandle);
2890
+ if (this.blurLocations.texelSizeLocation) {
2891
+ gl.uniform2f(this.blurLocations.texelSizeLocation, 1.0 / scaledWidth, 1.0 / scaledHeight);
2892
+ }
2893
+ const weights = this.getGaussianWeights(radius);
2894
+ if (this.blurLocations.weightsLocation) {
2895
+ gl.uniform1fv(this.blurLocations.weightsLocation, weights);
2896
+ }
2897
+ gl.viewport(0, 0, scaledWidth, scaledHeight);
2898
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this.pingFbo);
2899
+ gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
2900
+ if (this.blurLocations.directionLocation) {
2901
+ gl.uniform2f(this.blurLocations.directionLocation, 1.0, 0.0);
2902
+ }
2903
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
2904
+ gl.bindFramebuffer(gl.FRAMEBUFFER, this.pongFbo);
2905
+ gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
2906
+ if (this.blurLocations.directionLocation) {
2907
+ gl.uniform2f(this.blurLocations.directionLocation, 0.0, 1.0);
2908
+ }
2909
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
2910
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
2911
+ gl.viewport(0, 0, width, height);
2912
+ gl.useProgram(this.passthroughProgramHandle);
2913
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
2914
+ gl.enableVertexAttribArray(this.passthroughLocations.positionLocation);
2915
+ gl.vertexAttribPointer(this.passthroughLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
2916
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
2917
+ gl.enableVertexAttribArray(this.passthroughLocations.texCoordLocation);
2918
+ gl.vertexAttribPointer(this.passthroughLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
2919
+ gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
2920
+ if (this.passthroughLocations.imageLocation) {
2921
+ gl.uniform1i(this.passthroughLocations.imageLocation, 0);
2922
+ }
2923
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
2924
+ }
2925
+ close() {
2926
+ if (!this.isRunning)
2927
+ return;
2928
+ this.isRunning = false;
2929
+ const gl = this.gl;
2930
+ if (this.pingFbo)
2931
+ gl.deleteFramebuffer(this.pingFbo);
2932
+ if (this.pongFbo)
2933
+ gl.deleteFramebuffer(this.pongFbo);
2934
+ if (this.pingTexture)
2935
+ gl.deleteTexture(this.pingTexture);
2936
+ if (this.pongTexture)
2937
+ gl.deleteTexture(this.pongTexture);
2938
+ if (this.inputTexture)
2939
+ gl.deleteTexture(this.inputTexture);
2940
+ if (this.positionBuffer)
2941
+ gl.deleteBuffer(this.positionBuffer);
2942
+ if (this.texCoordBuffer)
2943
+ gl.deleteBuffer(this.texCoordBuffer);
2944
+ gl.deleteProgram(this.blurProgramHandle);
2945
+ gl.deleteProgram(this.passthroughProgramHandle);
2946
+ }
2947
+ }
2948
+
2949
+ /**
2950
+ * A video filter that applies a full-screen blur to each frame.
2951
+ *
2952
+ * It uses a WebGL renderer to blur the incoming camera track and outputs
2953
+ * a new track with the effect applied. Setup and frame handling are managed
2954
+ * by the base processor.
2955
+ */
2956
+ class FullScreenBlur extends BaseVideoProcessor {
2957
+ /**
2958
+ * Creates a new full-screen blur processor for the given video track.
2959
+ *
2960
+ * @param track - The input camera track to blur.
2961
+ * @param options - Optional settings such as the blur radius.
2962
+ * @param hooks - Optional callbacks for stats and error reporting.
2963
+ */
2964
+ constructor(track, options = {}, hooks = {}) {
2965
+ super(track, hooks);
2966
+ this.blurRadius = options.blurRadius ?? 6;
2967
+ }
2968
+ async initialize() {
2969
+ this.blurRenderer = new FullScreenBlurRenderer(this.canvas);
2970
+ }
2971
+ async transform(frame) {
2972
+ this.blurRenderer.render(frame, this.blurRadius);
2973
+ return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
2974
+ }
2975
+ onStop() {
2976
+ this.blurRenderer?.close();
2977
+ }
2978
+ get processorName() {
2979
+ return 'fullscreen-blur';
2980
+ }
2660
2981
  }
2661
2982
 
2662
- export { BACKGROUND_BLUR_MAP, SegmentationLevel, VirtualBackground, createRenderer, isMediaPipePlatformSupported, isPlatformSupported, loadMediaPipe, loadTFLite };
2983
+ export { BACKGROUND_BLUR_MAP, FullScreenBlur, SegmentationLevel, VirtualBackground, createRenderer, isMediaPipePlatformSupported, isPlatformSupported, loadMediaPipe, loadTFLite };
2663
2984
  //# sourceMappingURL=index.es.js.map