@stream-io/video-filters-web 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.cjs.js +449 -127
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.es.js +449 -128
- package/dist/index.es.js.map +1 -1
- package/dist/src/BaseVideoProcessor.d.ts +38 -0
- package/dist/src/FullScreenBlur.d.ts +28 -0
- package/dist/src/FullScreenBlurRenderer.d.ts +42 -0
- package/dist/src/VirtualBackground.d.ts +15 -28
- package/index.ts +1 -0
- package/package.json +1 -1
- package/src/BaseVideoProcessor.ts +132 -0
- package/src/FallbackProcessor.ts +4 -14
- package/src/FullScreenBlur.ts +52 -0
- package/src/FullScreenBlurRenderer.ts +411 -0
- package/src/VirtualBackground.ts +83 -184
package/dist/index.cjs.js
CHANGED
|
@@ -1561,7 +1561,7 @@ const createTFLiteSIMDModule = (__Module) => {
|
|
|
1561
1561
|
return __Module.ready;
|
|
1562
1562
|
};
|
|
1563
1563
|
|
|
1564
|
-
const version = "0.
|
|
1564
|
+
const version = "0.6.0";
|
|
1565
1565
|
const packageName = "@stream-io/video-filters-web";
|
|
1566
1566
|
|
|
1567
1567
|
// @ts-expect-error - module is not declared
|
|
@@ -2405,7 +2405,6 @@ class FallbackProcessor {
|
|
|
2405
2405
|
let timestamp = 0;
|
|
2406
2406
|
const frameRate = track.getSettings().frameRate || 30;
|
|
2407
2407
|
let frameDuration = 1000 / frameRate;
|
|
2408
|
-
let lastVideoTime = -1;
|
|
2409
2408
|
this.workerTimer = new workerTimer.WorkerTimer({ useWorker: true });
|
|
2410
2409
|
this.readable = new ReadableStream({
|
|
2411
2410
|
start: async () => {
|
|
@@ -2427,13 +2426,6 @@ class FallbackProcessor {
|
|
|
2427
2426
|
await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration - delta));
|
|
2428
2427
|
}
|
|
2429
2428
|
timestamp = performance.now();
|
|
2430
|
-
const currentTime = this.video.currentTime;
|
|
2431
|
-
const hasNewFrame = currentTime !== lastVideoTime;
|
|
2432
|
-
if (!hasNewFrame) {
|
|
2433
|
-
await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration));
|
|
2434
|
-
return;
|
|
2435
|
-
}
|
|
2436
|
-
lastVideoTime = currentTime;
|
|
2437
2429
|
if (canvas.width !== this.video.videoWidth ||
|
|
2438
2430
|
canvas.height !== this.video.videoHeight) {
|
|
2439
2431
|
canvas.width = this.video.videoWidth;
|
|
@@ -2441,7 +2433,9 @@ class FallbackProcessor {
|
|
|
2441
2433
|
}
|
|
2442
2434
|
ctx.drawImage(this.video, 0, 0);
|
|
2443
2435
|
try {
|
|
2444
|
-
const frame = new VideoFrame(canvas, {
|
|
2436
|
+
const frame = new VideoFrame(canvas, {
|
|
2437
|
+
timestamp: Math.round(this.video.currentTime * 1000000),
|
|
2438
|
+
});
|
|
2445
2439
|
controller.enqueue(frame);
|
|
2446
2440
|
}
|
|
2447
2441
|
catch (err) {
|
|
@@ -2462,93 +2456,134 @@ const TrackProcessor = typeof MediaStreamTrackProcessor !== 'undefined'
|
|
|
2462
2456
|
: FallbackProcessor;
|
|
2463
2457
|
|
|
2464
2458
|
/**
|
|
2465
|
-
*
|
|
2466
|
-
*
|
|
2467
|
-
*
|
|
2459
|
+
* Base class for real-time video filters.
|
|
2460
|
+
*
|
|
2461
|
+
* It sets up the full pipeline that reads frames from the input track,
|
|
2462
|
+
* processes them, and outputs a new track with your effect applied. Subclasses
|
|
2463
|
+
* only need to implement `initialize` (run once before processing starts) and
|
|
2464
|
+
* `transform` (called for every frame).
|
|
2465
|
+
*
|
|
2466
|
+
* Everything else—canvas setup, performance tracking, error handling, and
|
|
2467
|
+
* clean shutdown is handled for you. Calling `start()` returns a processed
|
|
2468
|
+
* `MediaStreamTrack` ready to use.
|
|
2468
2469
|
*/
|
|
2469
|
-
class
|
|
2470
|
-
|
|
2470
|
+
class BaseVideoProcessor {
|
|
2471
|
+
/**
|
|
2472
|
+
* Constructs a new instance.
|
|
2473
|
+
*/
|
|
2474
|
+
constructor(track, hooks = {}) {
|
|
2471
2475
|
this.track = track;
|
|
2472
|
-
this.
|
|
2473
|
-
this.hooks = hooks;
|
|
2474
|
-
this.segmenter = null;
|
|
2475
|
-
this.isSegmenterReady = false;
|
|
2476
|
-
this.segmenterDelayTotal = 0;
|
|
2476
|
+
this.abortController = new AbortController();
|
|
2477
2477
|
this.frames = 0;
|
|
2478
|
+
this.delayTotal = 0;
|
|
2478
2479
|
this.lastStatsTime = 0;
|
|
2479
2480
|
this.processor = new TrackProcessor({ track });
|
|
2480
2481
|
this.generator = new TrackGenerator({
|
|
2481
2482
|
kind: 'video',
|
|
2482
2483
|
signalTarget: track,
|
|
2483
2484
|
});
|
|
2484
|
-
this.
|
|
2485
|
+
this.hooks = hooks;
|
|
2485
2486
|
}
|
|
2486
2487
|
async start() {
|
|
2487
|
-
const { onError } = this.hooks;
|
|
2488
2488
|
const { readable } = this.processor;
|
|
2489
2489
|
const { writable } = this.generator;
|
|
2490
|
-
const
|
|
2491
|
-
|
|
2492
|
-
this.
|
|
2493
|
-
this.webGlRenderer = new WebGLRenderer(this.canvas);
|
|
2494
|
-
await this.initializeSegmenter();
|
|
2495
|
-
const opts = await this.initializeSegmenterOptions();
|
|
2490
|
+
const { width = 1280, height = 720 } = this.track.getSettings();
|
|
2491
|
+
this.canvas = new OffscreenCanvas(width, height);
|
|
2492
|
+
await this.initialize();
|
|
2496
2493
|
const transformStream = new TransformStream({
|
|
2497
2494
|
transform: async (frame, controller) => {
|
|
2498
2495
|
try {
|
|
2499
|
-
if (this.abortController.signal.aborted)
|
|
2496
|
+
if (this.abortController.signal.aborted)
|
|
2500
2497
|
return frame.close();
|
|
2498
|
+
if (this.canvas.width !== frame.displayWidth ||
|
|
2499
|
+
this.canvas.height !== frame.displayHeight) {
|
|
2500
|
+
this.canvas.width = frame.displayWidth;
|
|
2501
|
+
this.canvas.height = frame.displayHeight;
|
|
2501
2502
|
}
|
|
2502
|
-
const
|
|
2503
|
+
const start = performance.now();
|
|
2504
|
+
const processed = await this.transform(frame);
|
|
2505
|
+
const delay = performance.now() - start;
|
|
2506
|
+
this.updateStats(delay);
|
|
2503
2507
|
controller.enqueue(processed);
|
|
2504
2508
|
}
|
|
2505
2509
|
catch (e) {
|
|
2506
|
-
console.error('[virtual-background] error processing frame:', e);
|
|
2507
2510
|
this.hooks.onError?.(e);
|
|
2508
|
-
if (!this.abortController.signal.aborted) {
|
|
2509
|
-
controller.enqueue(frame);
|
|
2510
|
-
}
|
|
2511
2511
|
}
|
|
2512
2512
|
finally {
|
|
2513
2513
|
frame.close();
|
|
2514
2514
|
}
|
|
2515
2515
|
},
|
|
2516
|
-
flush: () =>
|
|
2517
|
-
if (this.segmenter) {
|
|
2518
|
-
this.segmenter.close();
|
|
2519
|
-
this.segmenter = null;
|
|
2520
|
-
}
|
|
2521
|
-
this.isSegmenterReady = false;
|
|
2522
|
-
},
|
|
2516
|
+
flush: () => this.onFlush(),
|
|
2523
2517
|
});
|
|
2524
|
-
const signal = this.abortController.signal;
|
|
2525
2518
|
readable
|
|
2526
|
-
.pipeThrough(transformStream, { signal })
|
|
2527
|
-
.pipeTo(writable)
|
|
2519
|
+
.pipeThrough(transformStream, { signal: this.abortController.signal })
|
|
2520
|
+
.pipeTo(writable, { signal: this.abortController.signal })
|
|
2528
2521
|
.catch((e) => {
|
|
2529
|
-
if (e.name !== 'AbortError') {
|
|
2530
|
-
console.error(
|
|
2531
|
-
onError?.(e);
|
|
2522
|
+
if (e.name !== 'AbortError' && e.name !== 'InvalidStateError') {
|
|
2523
|
+
console.error(`[${this.processorName}] Error processing track:`, e);
|
|
2524
|
+
this.hooks.onError?.(e);
|
|
2532
2525
|
}
|
|
2533
2526
|
});
|
|
2534
2527
|
return this.generator;
|
|
2535
2528
|
}
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2529
|
+
stop() {
|
|
2530
|
+
this.abortController.abort();
|
|
2531
|
+
this.generator.stop();
|
|
2532
|
+
this.onStop();
|
|
2533
|
+
}
|
|
2534
|
+
updateStats(delay) {
|
|
2535
|
+
this.frames++;
|
|
2536
|
+
this.delayTotal += delay;
|
|
2537
|
+
const now = performance.now();
|
|
2538
|
+
if (this.lastStatsTime === 0) {
|
|
2539
|
+
this.lastStatsTime = now;
|
|
2540
|
+
return;
|
|
2541
|
+
}
|
|
2542
|
+
if (now - this.lastStatsTime >= 1000) {
|
|
2543
|
+
const avgDelay = Math.round((this.delayTotal / this.frames) * 100) / 100;
|
|
2544
|
+
const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
|
|
2545
|
+
this.hooks.onStats?.({ delay: avgDelay, fps, timestamp: now });
|
|
2546
|
+
this.frames = 0;
|
|
2547
|
+
this.delayTotal = 0;
|
|
2548
|
+
this.lastStatsTime = now;
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2551
|
+
onFlush() { }
|
|
2552
|
+
onStop() { }
|
|
2553
|
+
get processorName() {
|
|
2554
|
+
return 'base-processor';
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
2557
|
+
|
|
2558
|
+
/**
|
|
2559
|
+
* Wraps a video MediaStreamTrack in a real-time processing pipeline.
|
|
2560
|
+
* Incoming frames are processed through a transformer and re-emitted
|
|
2561
|
+
* on a new MediaStreamVideoTrack for downstream consumption.
|
|
2562
|
+
*/
|
|
2563
|
+
class VirtualBackground extends BaseVideoProcessor {
|
|
2564
|
+
constructor(track, options = {}, hooks = {}) {
|
|
2565
|
+
super(track, hooks);
|
|
2566
|
+
this.options = options;
|
|
2567
|
+
this.segmenter = null;
|
|
2568
|
+
this.isSegmenterReady = false;
|
|
2569
|
+
this.latestCategoryMask = undefined;
|
|
2570
|
+
this.latestConfidenceMask = undefined;
|
|
2571
|
+
this.lastFrameTime = -1;
|
|
2572
|
+
this.count = 0;
|
|
2573
|
+
}
|
|
2574
|
+
async initialize() {
|
|
2575
|
+
this.webGlRenderer = new WebGLRenderer(this.canvas);
|
|
2576
|
+
await this.initializeSegmenter();
|
|
2577
|
+
}
|
|
2539
2578
|
async initializeSegmenter() {
|
|
2540
2579
|
try {
|
|
2541
|
-
|
|
2580
|
+
this.opts = await this.initializeSegmenterOptions();
|
|
2581
|
+
const basePath = this.options.basePath ||
|
|
2542
2582
|
`https://unpkg.com/${packageName}@${version}/mediapipe`;
|
|
2543
|
-
const
|
|
2544
|
-
const
|
|
2545
|
-
const wasmPath = `${basePath}/wasm`;
|
|
2546
|
-
const fileset = await tasksVision.FilesetResolver.forVisionTasks(wasmPath);
|
|
2583
|
+
const model = this.options.modelPath || `${basePath}/models/selfie_segmenter.tflite`;
|
|
2584
|
+
const fileset = await tasksVision.FilesetResolver.forVisionTasks(`${basePath}/wasm`);
|
|
2547
2585
|
this.segmenter = await tasksVision.ImageSegmenter.createFromOptions(fileset, {
|
|
2548
|
-
baseOptions: {
|
|
2549
|
-
modelAssetPath: model,
|
|
2550
|
-
delegate: 'GPU',
|
|
2551
|
-
},
|
|
2586
|
+
baseOptions: { modelAssetPath: model, delegate: 'GPU' },
|
|
2552
2587
|
runningMode: 'VIDEO',
|
|
2553
2588
|
outputCategoryMask: true,
|
|
2554
2589
|
outputConfidenceMasks: true,
|
|
@@ -2557,75 +2592,50 @@ class VirtualBackground {
|
|
|
2557
2592
|
this.isSegmenterReady = true;
|
|
2558
2593
|
}
|
|
2559
2594
|
catch (error) {
|
|
2560
|
-
console.error('[virtual-background]
|
|
2595
|
+
console.error('[virtual-background] Segmenter init failed:', error);
|
|
2561
2596
|
this.isSegmenterReady = false;
|
|
2562
2597
|
}
|
|
2563
2598
|
}
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
* @param opts - The segmentation options to use.
|
|
2572
|
-
*
|
|
2573
|
-
* @returns A new `VideoFrame` containing the processed image.
|
|
2574
|
-
*/
|
|
2575
|
-
async transform(frame, opts) {
|
|
2576
|
-
if (this.isSegmenterReady && this.segmenter) {
|
|
2577
|
-
try {
|
|
2578
|
-
const start = performance.now();
|
|
2579
|
-
await new Promise((resolve) => {
|
|
2580
|
-
this.segmenter.segmentForVideo(frame, frame.timestamp, (result) => {
|
|
2581
|
-
const categoryMask = result.categoryMask.getAsWebGLTexture();
|
|
2582
|
-
const confidenceMask = result.confidenceMasks[0].getAsWebGLTexture();
|
|
2583
|
-
this.webGlRenderer.render(frame, opts, categoryMask, confidenceMask);
|
|
2584
|
-
const now = performance.now();
|
|
2585
|
-
this.segmenterDelayTotal += now - start;
|
|
2586
|
-
this.frames++;
|
|
2587
|
-
if (this.lastStatsTime === 0) {
|
|
2588
|
-
this.lastStatsTime = now;
|
|
2589
|
-
}
|
|
2590
|
-
if (now - this.lastStatsTime > 1000) {
|
|
2591
|
-
const delay = Math.round((this.segmenterDelayTotal / this.frames) * 100) /
|
|
2592
|
-
100;
|
|
2593
|
-
const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
|
|
2594
|
-
this.hooks.onStats?.({ delay, fps, timestamp: now });
|
|
2595
|
-
this.lastStatsTime = now;
|
|
2596
|
-
this.segmenterDelayTotal = 0;
|
|
2597
|
-
this.frames = 0;
|
|
2598
|
-
}
|
|
2599
|
-
resolve();
|
|
2600
|
-
});
|
|
2601
|
-
});
|
|
2602
|
-
}
|
|
2603
|
-
catch (error) {
|
|
2604
|
-
console.error('[virtual-background] Error during segmentation:', error);
|
|
2605
|
-
}
|
|
2599
|
+
async transform(frame) {
|
|
2600
|
+
const currentTime = frame.timestamp;
|
|
2601
|
+
const hasNewFrame = currentTime !== this.lastFrameTime;
|
|
2602
|
+
this.lastFrameTime = currentTime;
|
|
2603
|
+
if (hasNewFrame && this.isSegmenterReady && this.segmenter) {
|
|
2604
|
+
await this.runSegmentation(frame);
|
|
2605
|
+
this.webGlRenderer.render(frame, this.opts, this.latestCategoryMask, this.latestConfidenceMask);
|
|
2606
2606
|
}
|
|
2607
2607
|
return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
|
|
2608
2608
|
}
|
|
2609
|
-
async
|
|
2610
|
-
if (!
|
|
2611
|
-
return;
|
|
2612
|
-
}
|
|
2613
|
-
const response = await fetch(url);
|
|
2614
|
-
if (!response.ok) {
|
|
2615
|
-
console.error(`[virtual-background] Failed to fetch background source ${url} (status: ${response.status})`);
|
|
2609
|
+
async runSegmentation(frame) {
|
|
2610
|
+
if (!this.segmenter)
|
|
2616
2611
|
return;
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2612
|
+
return new Promise((resolve) => {
|
|
2613
|
+
const timestamp = Math.floor(performance.now());
|
|
2614
|
+
this.segmenter.segmentForVideo(frame, timestamp, (result) => {
|
|
2615
|
+
try {
|
|
2616
|
+
this.latestCategoryMask = result.categoryMask?.getAsWebGLTexture();
|
|
2617
|
+
this.latestConfidenceMask =
|
|
2618
|
+
result.confidenceMasks?.[0]?.getAsWebGLTexture();
|
|
2619
|
+
}
|
|
2620
|
+
catch (err) {
|
|
2621
|
+
console.error('[virtual-background] segmentation error:', err);
|
|
2622
|
+
this.hooks.onError?.(err);
|
|
2623
|
+
}
|
|
2624
|
+
finally {
|
|
2625
|
+
result.close();
|
|
2626
|
+
resolve();
|
|
2627
|
+
}
|
|
2628
|
+
});
|
|
2629
|
+
});
|
|
2621
2630
|
}
|
|
2622
2631
|
async initializeSegmenterOptions() {
|
|
2623
2632
|
const isSelfieMode = this.options.modelPath
|
|
2624
|
-
? this.options.modelPath
|
|
2633
|
+
? this.options.modelPath.includes('selfie_segmenter')
|
|
2625
2634
|
: true;
|
|
2626
2635
|
if (this.options.backgroundFilter === 'image') {
|
|
2636
|
+
const source = await this.loadBackground(this.options.backgroundImage);
|
|
2627
2637
|
return {
|
|
2628
|
-
backgroundSource:
|
|
2638
|
+
backgroundSource: source,
|
|
2629
2639
|
bgBlur: 0,
|
|
2630
2640
|
bgBlurRadius: 0,
|
|
2631
2641
|
isSelfieMode,
|
|
@@ -2640,28 +2650,340 @@ class VirtualBackground {
|
|
|
2640
2650
|
};
|
|
2641
2651
|
}
|
|
2642
2652
|
const numeric = blurLevel ?? 5;
|
|
2643
|
-
const bgBlur = Math.min(numeric * 3, 30);
|
|
2644
|
-
const bgBlurRadius = Math.min(numeric, 10);
|
|
2645
2653
|
return {
|
|
2646
2654
|
backgroundSource: undefined,
|
|
2647
|
-
bgBlur,
|
|
2648
|
-
bgBlurRadius,
|
|
2655
|
+
bgBlur: Math.min(numeric * 3, 30),
|
|
2656
|
+
bgBlurRadius: Math.min(numeric, 10),
|
|
2649
2657
|
isSelfieMode,
|
|
2650
2658
|
};
|
|
2651
2659
|
}
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
if (
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
+
async loadBackground(url) {
|
|
2661
|
+
if (!url)
|
|
2662
|
+
return null;
|
|
2663
|
+
const result = await fetch(url);
|
|
2664
|
+
if (!result.ok)
|
|
2665
|
+
return null;
|
|
2666
|
+
return {
|
|
2667
|
+
type: 'image',
|
|
2668
|
+
media: await createImageBitmap(await result.blob()),
|
|
2669
|
+
url,
|
|
2670
|
+
};
|
|
2671
|
+
}
|
|
2672
|
+
onFlush() {
|
|
2673
|
+
this.destroySegmenter();
|
|
2674
|
+
}
|
|
2675
|
+
onStop() {
|
|
2676
|
+
this.webGlRenderer?.close();
|
|
2677
|
+
this.destroySegmenter();
|
|
2678
|
+
}
|
|
2679
|
+
destroySegmenter() {
|
|
2680
|
+
this.segmenter?.close();
|
|
2681
|
+
this.segmenter = null;
|
|
2660
2682
|
this.isSegmenterReady = false;
|
|
2661
2683
|
}
|
|
2684
|
+
get processorName() {
|
|
2685
|
+
return 'background-processor';
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
|
|
2689
|
+
/**
|
|
2690
|
+
* Simple WebGL renderer for full-screen Gaussian blur.
|
|
2691
|
+
* Uses a two-pass separable Gaussian blur (horizontal then vertical).
|
|
2692
|
+
* Optimized for moderation use cases by blurring at reduced resolution (15% scale)
|
|
2693
|
+
* and upscaling back to full resolution for output.
|
|
2694
|
+
*/
|
|
2695
|
+
class FullScreenBlurRenderer {
|
|
2696
|
+
constructor(canvas) {
|
|
2697
|
+
this.inputTexture = null;
|
|
2698
|
+
this.isRunning = false;
|
|
2699
|
+
this.targetWidth = 0;
|
|
2700
|
+
this.targetHeight = 0;
|
|
2701
|
+
this.weightCache = new Map();
|
|
2702
|
+
this.canvas = canvas;
|
|
2703
|
+
const gl = canvas.getContext('webgl2', {
|
|
2704
|
+
alpha: false,
|
|
2705
|
+
antialias: false,
|
|
2706
|
+
desynchronized: true,
|
|
2707
|
+
});
|
|
2708
|
+
if (!gl)
|
|
2709
|
+
throw new Error('WebGL2 not supported');
|
|
2710
|
+
this.gl = gl;
|
|
2711
|
+
const vertexShaderSource = `#version 300 es
|
|
2712
|
+
precision highp float;
|
|
2713
|
+
in vec2 a_position;
|
|
2714
|
+
in vec2 a_texCoord;
|
|
2715
|
+
out vec2 v_texCoord;
|
|
2716
|
+
void main() {
|
|
2717
|
+
v_texCoord = a_texCoord;
|
|
2718
|
+
gl_Position = vec4(a_position, 0.0, 1.0);
|
|
2719
|
+
}
|
|
2720
|
+
`;
|
|
2721
|
+
const fragmentShaderSource = `#version 300 es
|
|
2722
|
+
precision highp float;
|
|
2723
|
+
in vec2 v_texCoord;
|
|
2724
|
+
out vec4 outColor;
|
|
2725
|
+
uniform sampler2D u_image;
|
|
2726
|
+
uniform vec2 u_texelSize;
|
|
2727
|
+
uniform vec2 u_direction;
|
|
2728
|
+
uniform float u_weights[25];
|
|
2729
|
+
void main() {
|
|
2730
|
+
vec4 color = vec4(0.0);
|
|
2731
|
+
for (int i = -12; i <= 12; i++) {
|
|
2732
|
+
float w = u_weights[i + 12];
|
|
2733
|
+
if (w == 0.0) continue;
|
|
2734
|
+
vec2 offset = float(i) * u_direction * u_texelSize;
|
|
2735
|
+
color += w * texture(u_image, v_texCoord + offset);
|
|
2736
|
+
}
|
|
2737
|
+
outColor = color;
|
|
2738
|
+
}
|
|
2739
|
+
`;
|
|
2740
|
+
this.blurProgramHandle = this.createAndLinkProgram(vertexShaderSource, fragmentShaderSource);
|
|
2741
|
+
const passthroughFragmentShaderSource = `#version 300 es
|
|
2742
|
+
precision highp float;
|
|
2743
|
+
in vec2 v_texCoord;
|
|
2744
|
+
out vec4 outColor;
|
|
2745
|
+
uniform sampler2D u_image;
|
|
2746
|
+
void main() {
|
|
2747
|
+
outColor = texture(u_image, v_texCoord);
|
|
2748
|
+
}
|
|
2749
|
+
`;
|
|
2750
|
+
this.passthroughProgramHandle = this.createAndLinkProgram(vertexShaderSource, passthroughFragmentShaderSource);
|
|
2751
|
+
const blurProgram = this.blurProgramHandle;
|
|
2752
|
+
const passthroughProgram = this.passthroughProgramHandle;
|
|
2753
|
+
this.blurLocations = {
|
|
2754
|
+
positionLocation: gl.getAttribLocation(blurProgram, 'a_position'),
|
|
2755
|
+
texCoordLocation: gl.getAttribLocation(blurProgram, 'a_texCoord'),
|
|
2756
|
+
imageLocation: gl.getUniformLocation(blurProgram, 'u_image'),
|
|
2757
|
+
texelSizeLocation: gl.getUniformLocation(blurProgram, 'u_texelSize'),
|
|
2758
|
+
directionLocation: gl.getUniformLocation(blurProgram, 'u_direction'),
|
|
2759
|
+
weightsLocation: gl.getUniformLocation(blurProgram, 'u_weights'),
|
|
2760
|
+
};
|
|
2761
|
+
this.passthroughLocations = {
|
|
2762
|
+
positionLocation: gl.getAttribLocation(passthroughProgram, 'a_position'),
|
|
2763
|
+
texCoordLocation: gl.getAttribLocation(passthroughProgram, 'a_texCoord'),
|
|
2764
|
+
imageLocation: gl.getUniformLocation(passthroughProgram, 'u_image'),
|
|
2765
|
+
};
|
|
2766
|
+
this.positionBuffer = gl.createBuffer();
|
|
2767
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2768
|
+
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
|
|
2769
|
+
this.texCoordBuffer = gl.createBuffer();
|
|
2770
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2771
|
+
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
|
|
2772
|
+
const createTexture2D = () => {
|
|
2773
|
+
const tex = gl.createTexture();
|
|
2774
|
+
if (!tex)
|
|
2775
|
+
throw new Error('Failed to create texture');
|
|
2776
|
+
gl.bindTexture(gl.TEXTURE_2D, tex);
|
|
2777
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2778
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2779
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
2780
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
|
2781
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2782
|
+
return tex;
|
|
2783
|
+
};
|
|
2784
|
+
const createFramebufferForTexture = (tex) => {
|
|
2785
|
+
const fb = gl.createFramebuffer();
|
|
2786
|
+
if (!fb)
|
|
2787
|
+
throw new Error('Failed to create framebuffer');
|
|
2788
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
|
|
2789
|
+
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
|
|
2790
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2791
|
+
return fb;
|
|
2792
|
+
};
|
|
2793
|
+
this.pingTexture = createTexture2D();
|
|
2794
|
+
this.pongTexture = createTexture2D();
|
|
2795
|
+
this.pingFbo = createFramebufferForTexture(this.pingTexture);
|
|
2796
|
+
this.pongFbo = createFramebufferForTexture(this.pongTexture);
|
|
2797
|
+
this.inputTexture = createTexture2D();
|
|
2798
|
+
gl.useProgram(blurProgram);
|
|
2799
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2800
|
+
gl.enableVertexAttribArray(this.blurLocations.positionLocation);
|
|
2801
|
+
gl.vertexAttribPointer(this.blurLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2802
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2803
|
+
gl.enableVertexAttribArray(this.blurLocations.texCoordLocation);
|
|
2804
|
+
gl.vertexAttribPointer(this.blurLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2805
|
+
if (this.blurLocations.imageLocation) {
|
|
2806
|
+
gl.uniform1i(this.blurLocations.imageLocation, 0);
|
|
2807
|
+
}
|
|
2808
|
+
this.isRunning = true;
|
|
2809
|
+
}
|
|
2810
|
+
createAndLinkProgram(vsSource, fsSource) {
|
|
2811
|
+
const gl = this.gl;
|
|
2812
|
+
const vs = this.createShader(gl.VERTEX_SHADER, vsSource);
|
|
2813
|
+
const fs = this.createShader(gl.FRAGMENT_SHADER, fsSource);
|
|
2814
|
+
const prog = gl.createProgram();
|
|
2815
|
+
if (!prog)
|
|
2816
|
+
throw new Error('Failed to create program');
|
|
2817
|
+
gl.attachShader(prog, vs);
|
|
2818
|
+
gl.attachShader(prog, fs);
|
|
2819
|
+
gl.linkProgram(prog);
|
|
2820
|
+
if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
|
|
2821
|
+
throw new Error('Shader link failed: ' + gl.getProgramInfoLog(prog));
|
|
2822
|
+
}
|
|
2823
|
+
gl.deleteShader(vs);
|
|
2824
|
+
gl.deleteShader(fs);
|
|
2825
|
+
return prog;
|
|
2826
|
+
}
|
|
2827
|
+
createShader(type, source) {
|
|
2828
|
+
const gl = this.gl;
|
|
2829
|
+
const shader = gl.createShader(type);
|
|
2830
|
+
if (!shader)
|
|
2831
|
+
throw new Error('Failed to create shader');
|
|
2832
|
+
gl.shaderSource(shader, source);
|
|
2833
|
+
gl.compileShader(shader);
|
|
2834
|
+
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
|
2835
|
+
throw new Error('Shader compile failed: ' + gl.getShaderInfoLog(shader));
|
|
2836
|
+
}
|
|
2837
|
+
return shader;
|
|
2838
|
+
}
|
|
2839
|
+
getGaussianWeights(radius) {
|
|
2840
|
+
const r = Math.max(0, Math.min(radius | 0, 12));
|
|
2841
|
+
const cached = this.weightCache.get(r);
|
|
2842
|
+
if (cached)
|
|
2843
|
+
return cached;
|
|
2844
|
+
const weights = new Float32Array(25);
|
|
2845
|
+
if (r === 0) {
|
|
2846
|
+
weights[12] = 1.0;
|
|
2847
|
+
this.weightCache.set(r, weights);
|
|
2848
|
+
return weights;
|
|
2849
|
+
}
|
|
2850
|
+
const sigma = r * 0.6;
|
|
2851
|
+
let sum = 0;
|
|
2852
|
+
for (let i = -r; i <= r; i++) {
|
|
2853
|
+
const w = Math.exp(-(i * i) / (2 * sigma * sigma));
|
|
2854
|
+
weights[i + 12] = w;
|
|
2855
|
+
sum += w;
|
|
2856
|
+
}
|
|
2857
|
+
for (let i = -r; i <= r; i++) {
|
|
2858
|
+
weights[i + 12] /= sum;
|
|
2859
|
+
}
|
|
2860
|
+
this.weightCache.set(r, weights);
|
|
2861
|
+
return weights;
|
|
2862
|
+
}
|
|
2863
|
+
render(frame, radius) {
|
|
2864
|
+
if (!this.isRunning)
|
|
2865
|
+
return;
|
|
2866
|
+
const gl = this.gl;
|
|
2867
|
+
const width = frame.displayWidth;
|
|
2868
|
+
const height = frame.displayHeight;
|
|
2869
|
+
if (!width || !height)
|
|
2870
|
+
return;
|
|
2871
|
+
if (this.canvas.width !== width || this.canvas.height !== height) {
|
|
2872
|
+
this.canvas.width = width;
|
|
2873
|
+
this.canvas.height = height;
|
|
2874
|
+
}
|
|
2875
|
+
const scale = 0.15;
|
|
2876
|
+
const scaledWidth = Math.max(1, Math.floor(width * scale));
|
|
2877
|
+
const scaledHeight = Math.max(1, Math.floor(height * scale));
|
|
2878
|
+
if (scaledWidth !== this.targetWidth ||
|
|
2879
|
+
scaledHeight !== this.targetHeight) {
|
|
2880
|
+
this.targetWidth = scaledWidth;
|
|
2881
|
+
this.targetHeight = scaledHeight;
|
|
2882
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
|
|
2883
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2884
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
|
|
2885
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2886
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2887
|
+
}
|
|
2888
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
2889
|
+
gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
|
|
2890
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
|
|
2891
|
+
gl.useProgram(this.blurProgramHandle);
|
|
2892
|
+
if (this.blurLocations.texelSizeLocation) {
|
|
2893
|
+
gl.uniform2f(this.blurLocations.texelSizeLocation, 1.0 / scaledWidth, 1.0 / scaledHeight);
|
|
2894
|
+
}
|
|
2895
|
+
const weights = this.getGaussianWeights(radius);
|
|
2896
|
+
if (this.blurLocations.weightsLocation) {
|
|
2897
|
+
gl.uniform1fv(this.blurLocations.weightsLocation, weights);
|
|
2898
|
+
}
|
|
2899
|
+
gl.viewport(0, 0, scaledWidth, scaledHeight);
|
|
2900
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, this.pingFbo);
|
|
2901
|
+
gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
|
|
2902
|
+
if (this.blurLocations.directionLocation) {
|
|
2903
|
+
gl.uniform2f(this.blurLocations.directionLocation, 1.0, 0.0);
|
|
2904
|
+
}
|
|
2905
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2906
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, this.pongFbo);
|
|
2907
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
|
|
2908
|
+
if (this.blurLocations.directionLocation) {
|
|
2909
|
+
gl.uniform2f(this.blurLocations.directionLocation, 0.0, 1.0);
|
|
2910
|
+
}
|
|
2911
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2912
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2913
|
+
gl.viewport(0, 0, width, height);
|
|
2914
|
+
gl.useProgram(this.passthroughProgramHandle);
|
|
2915
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2916
|
+
gl.enableVertexAttribArray(this.passthroughLocations.positionLocation);
|
|
2917
|
+
gl.vertexAttribPointer(this.passthroughLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2918
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2919
|
+
gl.enableVertexAttribArray(this.passthroughLocations.texCoordLocation);
|
|
2920
|
+
gl.vertexAttribPointer(this.passthroughLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2921
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
|
|
2922
|
+
if (this.passthroughLocations.imageLocation) {
|
|
2923
|
+
gl.uniform1i(this.passthroughLocations.imageLocation, 0);
|
|
2924
|
+
}
|
|
2925
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2926
|
+
}
|
|
2927
|
+
close() {
|
|
2928
|
+
if (!this.isRunning)
|
|
2929
|
+
return;
|
|
2930
|
+
this.isRunning = false;
|
|
2931
|
+
const gl = this.gl;
|
|
2932
|
+
if (this.pingFbo)
|
|
2933
|
+
gl.deleteFramebuffer(this.pingFbo);
|
|
2934
|
+
if (this.pongFbo)
|
|
2935
|
+
gl.deleteFramebuffer(this.pongFbo);
|
|
2936
|
+
if (this.pingTexture)
|
|
2937
|
+
gl.deleteTexture(this.pingTexture);
|
|
2938
|
+
if (this.pongTexture)
|
|
2939
|
+
gl.deleteTexture(this.pongTexture);
|
|
2940
|
+
if (this.inputTexture)
|
|
2941
|
+
gl.deleteTexture(this.inputTexture);
|
|
2942
|
+
if (this.positionBuffer)
|
|
2943
|
+
gl.deleteBuffer(this.positionBuffer);
|
|
2944
|
+
if (this.texCoordBuffer)
|
|
2945
|
+
gl.deleteBuffer(this.texCoordBuffer);
|
|
2946
|
+
gl.deleteProgram(this.blurProgramHandle);
|
|
2947
|
+
gl.deleteProgram(this.passthroughProgramHandle);
|
|
2948
|
+
}
|
|
2949
|
+
}
|
|
2950
|
+
|
|
2951
|
+
/**
|
|
2952
|
+
* A video filter that applies a full-screen blur to each frame.
|
|
2953
|
+
*
|
|
2954
|
+
* It uses a WebGL renderer to blur the incoming camera track and outputs
|
|
2955
|
+
* a new track with the effect applied. Setup and frame handling are managed
|
|
2956
|
+
* by the base processor.
|
|
2957
|
+
*/
|
|
2958
|
+
class FullScreenBlur extends BaseVideoProcessor {
|
|
2959
|
+
/**
|
|
2960
|
+
* Creates a new full-screen blur processor for the given video track.
|
|
2961
|
+
*
|
|
2962
|
+
* @param track - The input camera track to blur.
|
|
2963
|
+
* @param options - Optional settings such as the blur radius.
|
|
2964
|
+
* @param hooks - Optional callbacks for stats and error reporting.
|
|
2965
|
+
*/
|
|
2966
|
+
constructor(track, options = {}, hooks = {}) {
|
|
2967
|
+
super(track, hooks);
|
|
2968
|
+
this.blurRadius = options.blurRadius ?? 6;
|
|
2969
|
+
}
|
|
2970
|
+
async initialize() {
|
|
2971
|
+
this.blurRenderer = new FullScreenBlurRenderer(this.canvas);
|
|
2972
|
+
}
|
|
2973
|
+
async transform(frame) {
|
|
2974
|
+
this.blurRenderer.render(frame, this.blurRadius);
|
|
2975
|
+
return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
|
|
2976
|
+
}
|
|
2977
|
+
onStop() {
|
|
2978
|
+
this.blurRenderer?.close();
|
|
2979
|
+
}
|
|
2980
|
+
get processorName() {
|
|
2981
|
+
return 'fullscreen-blur';
|
|
2982
|
+
}
|
|
2662
2983
|
}
|
|
2663
2984
|
|
|
2664
2985
|
exports.BACKGROUND_BLUR_MAP = BACKGROUND_BLUR_MAP;
|
|
2986
|
+
exports.FullScreenBlur = FullScreenBlur;
|
|
2665
2987
|
exports.VirtualBackground = VirtualBackground;
|
|
2666
2988
|
exports.createRenderer = createRenderer;
|
|
2667
2989
|
exports.isMediaPipePlatformSupported = isMediaPipePlatformSupported;
|