@stream-io/video-filters-web 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.cjs.js +449 -127
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.es.js +449 -128
- package/dist/index.es.js.map +1 -1
- package/dist/src/BaseVideoProcessor.d.ts +38 -0
- package/dist/src/FullScreenBlur.d.ts +28 -0
- package/dist/src/FullScreenBlurRenderer.d.ts +42 -0
- package/dist/src/VirtualBackground.d.ts +15 -28
- package/index.ts +1 -0
- package/package.json +1 -1
- package/src/BaseVideoProcessor.ts +132 -0
- package/src/FallbackProcessor.ts +4 -14
- package/src/FullScreenBlur.ts +52 -0
- package/src/FullScreenBlurRenderer.ts +411 -0
- package/src/VirtualBackground.ts +83 -184
package/dist/index.d.ts
CHANGED
package/dist/index.es.js
CHANGED
|
@@ -1559,7 +1559,7 @@ const createTFLiteSIMDModule = (__Module) => {
|
|
|
1559
1559
|
return __Module.ready;
|
|
1560
1560
|
};
|
|
1561
1561
|
|
|
1562
|
-
const version = "0.
|
|
1562
|
+
const version = "0.6.0";
|
|
1563
1563
|
const packageName = "@stream-io/video-filters-web";
|
|
1564
1564
|
|
|
1565
1565
|
// @ts-expect-error - module is not declared
|
|
@@ -2403,7 +2403,6 @@ class FallbackProcessor {
|
|
|
2403
2403
|
let timestamp = 0;
|
|
2404
2404
|
const frameRate = track.getSettings().frameRate || 30;
|
|
2405
2405
|
let frameDuration = 1000 / frameRate;
|
|
2406
|
-
let lastVideoTime = -1;
|
|
2407
2406
|
this.workerTimer = new WorkerTimer({ useWorker: true });
|
|
2408
2407
|
this.readable = new ReadableStream({
|
|
2409
2408
|
start: async () => {
|
|
@@ -2425,13 +2424,6 @@ class FallbackProcessor {
|
|
|
2425
2424
|
await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration - delta));
|
|
2426
2425
|
}
|
|
2427
2426
|
timestamp = performance.now();
|
|
2428
|
-
const currentTime = this.video.currentTime;
|
|
2429
|
-
const hasNewFrame = currentTime !== lastVideoTime;
|
|
2430
|
-
if (!hasNewFrame) {
|
|
2431
|
-
await new Promise((r) => this.workerTimer.setTimeout(r, frameDuration));
|
|
2432
|
-
return;
|
|
2433
|
-
}
|
|
2434
|
-
lastVideoTime = currentTime;
|
|
2435
2427
|
if (canvas.width !== this.video.videoWidth ||
|
|
2436
2428
|
canvas.height !== this.video.videoHeight) {
|
|
2437
2429
|
canvas.width = this.video.videoWidth;
|
|
@@ -2439,7 +2431,9 @@ class FallbackProcessor {
|
|
|
2439
2431
|
}
|
|
2440
2432
|
ctx.drawImage(this.video, 0, 0);
|
|
2441
2433
|
try {
|
|
2442
|
-
const frame = new VideoFrame(canvas, {
|
|
2434
|
+
const frame = new VideoFrame(canvas, {
|
|
2435
|
+
timestamp: Math.round(this.video.currentTime * 1000000),
|
|
2436
|
+
});
|
|
2443
2437
|
controller.enqueue(frame);
|
|
2444
2438
|
}
|
|
2445
2439
|
catch (err) {
|
|
@@ -2460,93 +2454,134 @@ const TrackProcessor = typeof MediaStreamTrackProcessor !== 'undefined'
|
|
|
2460
2454
|
: FallbackProcessor;
|
|
2461
2455
|
|
|
2462
2456
|
/**
|
|
2463
|
-
*
|
|
2464
|
-
*
|
|
2465
|
-
*
|
|
2457
|
+
* Base class for real-time video filters.
|
|
2458
|
+
*
|
|
2459
|
+
* It sets up the full pipeline that reads frames from the input track,
|
|
2460
|
+
* processes them, and outputs a new track with your effect applied. Subclasses
|
|
2461
|
+
* only need to implement `initialize` (run once before processing starts) and
|
|
2462
|
+
* `transform` (called for every frame).
|
|
2463
|
+
*
|
|
2464
|
+
* Everything else—canvas setup, performance tracking, error handling, and
|
|
2465
|
+
* clean shutdown is handled for you. Calling `start()` returns a processed
|
|
2466
|
+
* `MediaStreamTrack` ready to use.
|
|
2466
2467
|
*/
|
|
2467
|
-
class
|
|
2468
|
-
|
|
2468
|
+
class BaseVideoProcessor {
|
|
2469
|
+
/**
|
|
2470
|
+
* Constructs a new instance.
|
|
2471
|
+
*/
|
|
2472
|
+
constructor(track, hooks = {}) {
|
|
2469
2473
|
this.track = track;
|
|
2470
|
-
this.
|
|
2471
|
-
this.hooks = hooks;
|
|
2472
|
-
this.segmenter = null;
|
|
2473
|
-
this.isSegmenterReady = false;
|
|
2474
|
-
this.segmenterDelayTotal = 0;
|
|
2474
|
+
this.abortController = new AbortController();
|
|
2475
2475
|
this.frames = 0;
|
|
2476
|
+
this.delayTotal = 0;
|
|
2476
2477
|
this.lastStatsTime = 0;
|
|
2477
2478
|
this.processor = new TrackProcessor({ track });
|
|
2478
2479
|
this.generator = new TrackGenerator({
|
|
2479
2480
|
kind: 'video',
|
|
2480
2481
|
signalTarget: track,
|
|
2481
2482
|
});
|
|
2482
|
-
this.
|
|
2483
|
+
this.hooks = hooks;
|
|
2483
2484
|
}
|
|
2484
2485
|
async start() {
|
|
2485
|
-
const { onError } = this.hooks;
|
|
2486
2486
|
const { readable } = this.processor;
|
|
2487
2487
|
const { writable } = this.generator;
|
|
2488
|
-
const
|
|
2489
|
-
|
|
2490
|
-
this.
|
|
2491
|
-
this.webGlRenderer = new WebGLRenderer(this.canvas);
|
|
2492
|
-
await this.initializeSegmenter();
|
|
2493
|
-
const opts = await this.initializeSegmenterOptions();
|
|
2488
|
+
const { width = 1280, height = 720 } = this.track.getSettings();
|
|
2489
|
+
this.canvas = new OffscreenCanvas(width, height);
|
|
2490
|
+
await this.initialize();
|
|
2494
2491
|
const transformStream = new TransformStream({
|
|
2495
2492
|
transform: async (frame, controller) => {
|
|
2496
2493
|
try {
|
|
2497
|
-
if (this.abortController.signal.aborted)
|
|
2494
|
+
if (this.abortController.signal.aborted)
|
|
2498
2495
|
return frame.close();
|
|
2496
|
+
if (this.canvas.width !== frame.displayWidth ||
|
|
2497
|
+
this.canvas.height !== frame.displayHeight) {
|
|
2498
|
+
this.canvas.width = frame.displayWidth;
|
|
2499
|
+
this.canvas.height = frame.displayHeight;
|
|
2499
2500
|
}
|
|
2500
|
-
const
|
|
2501
|
+
const start = performance.now();
|
|
2502
|
+
const processed = await this.transform(frame);
|
|
2503
|
+
const delay = performance.now() - start;
|
|
2504
|
+
this.updateStats(delay);
|
|
2501
2505
|
controller.enqueue(processed);
|
|
2502
2506
|
}
|
|
2503
2507
|
catch (e) {
|
|
2504
|
-
console.error('[virtual-background] error processing frame:', e);
|
|
2505
2508
|
this.hooks.onError?.(e);
|
|
2506
|
-
if (!this.abortController.signal.aborted) {
|
|
2507
|
-
controller.enqueue(frame);
|
|
2508
|
-
}
|
|
2509
2509
|
}
|
|
2510
2510
|
finally {
|
|
2511
2511
|
frame.close();
|
|
2512
2512
|
}
|
|
2513
2513
|
},
|
|
2514
|
-
flush: () =>
|
|
2515
|
-
if (this.segmenter) {
|
|
2516
|
-
this.segmenter.close();
|
|
2517
|
-
this.segmenter = null;
|
|
2518
|
-
}
|
|
2519
|
-
this.isSegmenterReady = false;
|
|
2520
|
-
},
|
|
2514
|
+
flush: () => this.onFlush(),
|
|
2521
2515
|
});
|
|
2522
|
-
const signal = this.abortController.signal;
|
|
2523
2516
|
readable
|
|
2524
|
-
.pipeThrough(transformStream, { signal })
|
|
2525
|
-
.pipeTo(writable)
|
|
2517
|
+
.pipeThrough(transformStream, { signal: this.abortController.signal })
|
|
2518
|
+
.pipeTo(writable, { signal: this.abortController.signal })
|
|
2526
2519
|
.catch((e) => {
|
|
2527
|
-
if (e.name !== 'AbortError') {
|
|
2528
|
-
console.error(
|
|
2529
|
-
onError?.(e);
|
|
2520
|
+
if (e.name !== 'AbortError' && e.name !== 'InvalidStateError') {
|
|
2521
|
+
console.error(`[${this.processorName}] Error processing track:`, e);
|
|
2522
|
+
this.hooks.onError?.(e);
|
|
2530
2523
|
}
|
|
2531
2524
|
});
|
|
2532
2525
|
return this.generator;
|
|
2533
2526
|
}
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2527
|
+
stop() {
|
|
2528
|
+
this.abortController.abort();
|
|
2529
|
+
this.generator.stop();
|
|
2530
|
+
this.onStop();
|
|
2531
|
+
}
|
|
2532
|
+
updateStats(delay) {
|
|
2533
|
+
this.frames++;
|
|
2534
|
+
this.delayTotal += delay;
|
|
2535
|
+
const now = performance.now();
|
|
2536
|
+
if (this.lastStatsTime === 0) {
|
|
2537
|
+
this.lastStatsTime = now;
|
|
2538
|
+
return;
|
|
2539
|
+
}
|
|
2540
|
+
if (now - this.lastStatsTime >= 1000) {
|
|
2541
|
+
const avgDelay = Math.round((this.delayTotal / this.frames) * 100) / 100;
|
|
2542
|
+
const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
|
|
2543
|
+
this.hooks.onStats?.({ delay: avgDelay, fps, timestamp: now });
|
|
2544
|
+
this.frames = 0;
|
|
2545
|
+
this.delayTotal = 0;
|
|
2546
|
+
this.lastStatsTime = now;
|
|
2547
|
+
}
|
|
2548
|
+
}
|
|
2549
|
+
onFlush() { }
|
|
2550
|
+
onStop() { }
|
|
2551
|
+
get processorName() {
|
|
2552
|
+
return 'base-processor';
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
|
|
2556
|
+
/**
|
|
2557
|
+
* Wraps a video MediaStreamTrack in a real-time processing pipeline.
|
|
2558
|
+
* Incoming frames are processed through a transformer and re-emitted
|
|
2559
|
+
* on a new MediaStreamVideoTrack for downstream consumption.
|
|
2560
|
+
*/
|
|
2561
|
+
class VirtualBackground extends BaseVideoProcessor {
|
|
2562
|
+
constructor(track, options = {}, hooks = {}) {
|
|
2563
|
+
super(track, hooks);
|
|
2564
|
+
this.options = options;
|
|
2565
|
+
this.segmenter = null;
|
|
2566
|
+
this.isSegmenterReady = false;
|
|
2567
|
+
this.latestCategoryMask = undefined;
|
|
2568
|
+
this.latestConfidenceMask = undefined;
|
|
2569
|
+
this.lastFrameTime = -1;
|
|
2570
|
+
this.count = 0;
|
|
2571
|
+
}
|
|
2572
|
+
async initialize() {
|
|
2573
|
+
this.webGlRenderer = new WebGLRenderer(this.canvas);
|
|
2574
|
+
await this.initializeSegmenter();
|
|
2575
|
+
}
|
|
2537
2576
|
async initializeSegmenter() {
|
|
2538
2577
|
try {
|
|
2539
|
-
|
|
2578
|
+
this.opts = await this.initializeSegmenterOptions();
|
|
2579
|
+
const basePath = this.options.basePath ||
|
|
2540
2580
|
`https://unpkg.com/${packageName}@${version}/mediapipe`;
|
|
2541
|
-
const
|
|
2542
|
-
const
|
|
2543
|
-
const wasmPath = `${basePath}/wasm`;
|
|
2544
|
-
const fileset = await FilesetResolver.forVisionTasks(wasmPath);
|
|
2581
|
+
const model = this.options.modelPath || `${basePath}/models/selfie_segmenter.tflite`;
|
|
2582
|
+
const fileset = await FilesetResolver.forVisionTasks(`${basePath}/wasm`);
|
|
2545
2583
|
this.segmenter = await ImageSegmenter.createFromOptions(fileset, {
|
|
2546
|
-
baseOptions: {
|
|
2547
|
-
modelAssetPath: model,
|
|
2548
|
-
delegate: 'GPU',
|
|
2549
|
-
},
|
|
2584
|
+
baseOptions: { modelAssetPath: model, delegate: 'GPU' },
|
|
2550
2585
|
runningMode: 'VIDEO',
|
|
2551
2586
|
outputCategoryMask: true,
|
|
2552
2587
|
outputConfidenceMasks: true,
|
|
@@ -2555,75 +2590,50 @@ class VirtualBackground {
|
|
|
2555
2590
|
this.isSegmenterReady = true;
|
|
2556
2591
|
}
|
|
2557
2592
|
catch (error) {
|
|
2558
|
-
console.error('[virtual-background]
|
|
2593
|
+
console.error('[virtual-background] Segmenter init failed:', error);
|
|
2559
2594
|
this.isSegmenterReady = false;
|
|
2560
2595
|
}
|
|
2561
2596
|
}
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
* @param opts - The segmentation options to use.
|
|
2570
|
-
*
|
|
2571
|
-
* @returns A new `VideoFrame` containing the processed image.
|
|
2572
|
-
*/
|
|
2573
|
-
async transform(frame, opts) {
|
|
2574
|
-
if (this.isSegmenterReady && this.segmenter) {
|
|
2575
|
-
try {
|
|
2576
|
-
const start = performance.now();
|
|
2577
|
-
await new Promise((resolve) => {
|
|
2578
|
-
this.segmenter.segmentForVideo(frame, frame.timestamp, (result) => {
|
|
2579
|
-
const categoryMask = result.categoryMask.getAsWebGLTexture();
|
|
2580
|
-
const confidenceMask = result.confidenceMasks[0].getAsWebGLTexture();
|
|
2581
|
-
this.webGlRenderer.render(frame, opts, categoryMask, confidenceMask);
|
|
2582
|
-
const now = performance.now();
|
|
2583
|
-
this.segmenterDelayTotal += now - start;
|
|
2584
|
-
this.frames++;
|
|
2585
|
-
if (this.lastStatsTime === 0) {
|
|
2586
|
-
this.lastStatsTime = now;
|
|
2587
|
-
}
|
|
2588
|
-
if (now - this.lastStatsTime > 1000) {
|
|
2589
|
-
const delay = Math.round((this.segmenterDelayTotal / this.frames) * 100) /
|
|
2590
|
-
100;
|
|
2591
|
-
const fps = Math.round((1000 * this.frames) / (now - this.lastStatsTime));
|
|
2592
|
-
this.hooks.onStats?.({ delay, fps, timestamp: now });
|
|
2593
|
-
this.lastStatsTime = now;
|
|
2594
|
-
this.segmenterDelayTotal = 0;
|
|
2595
|
-
this.frames = 0;
|
|
2596
|
-
}
|
|
2597
|
-
resolve();
|
|
2598
|
-
});
|
|
2599
|
-
});
|
|
2600
|
-
}
|
|
2601
|
-
catch (error) {
|
|
2602
|
-
console.error('[virtual-background] Error during segmentation:', error);
|
|
2603
|
-
}
|
|
2597
|
+
async transform(frame) {
|
|
2598
|
+
const currentTime = frame.timestamp;
|
|
2599
|
+
const hasNewFrame = currentTime !== this.lastFrameTime;
|
|
2600
|
+
this.lastFrameTime = currentTime;
|
|
2601
|
+
if (hasNewFrame && this.isSegmenterReady && this.segmenter) {
|
|
2602
|
+
await this.runSegmentation(frame);
|
|
2603
|
+
this.webGlRenderer.render(frame, this.opts, this.latestCategoryMask, this.latestConfidenceMask);
|
|
2604
2604
|
}
|
|
2605
2605
|
return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
|
|
2606
2606
|
}
|
|
2607
|
-
async
|
|
2608
|
-
if (!
|
|
2609
|
-
return;
|
|
2610
|
-
}
|
|
2611
|
-
const response = await fetch(url);
|
|
2612
|
-
if (!response.ok) {
|
|
2613
|
-
console.error(`[virtual-background] Failed to fetch background source ${url} (status: ${response.status})`);
|
|
2607
|
+
async runSegmentation(frame) {
|
|
2608
|
+
if (!this.segmenter)
|
|
2614
2609
|
return;
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2610
|
+
return new Promise((resolve) => {
|
|
2611
|
+
const timestamp = Math.floor(performance.now());
|
|
2612
|
+
this.segmenter.segmentForVideo(frame, timestamp, (result) => {
|
|
2613
|
+
try {
|
|
2614
|
+
this.latestCategoryMask = result.categoryMask?.getAsWebGLTexture();
|
|
2615
|
+
this.latestConfidenceMask =
|
|
2616
|
+
result.confidenceMasks?.[0]?.getAsWebGLTexture();
|
|
2617
|
+
}
|
|
2618
|
+
catch (err) {
|
|
2619
|
+
console.error('[virtual-background] segmentation error:', err);
|
|
2620
|
+
this.hooks.onError?.(err);
|
|
2621
|
+
}
|
|
2622
|
+
finally {
|
|
2623
|
+
result.close();
|
|
2624
|
+
resolve();
|
|
2625
|
+
}
|
|
2626
|
+
});
|
|
2627
|
+
});
|
|
2619
2628
|
}
|
|
2620
2629
|
async initializeSegmenterOptions() {
|
|
2621
2630
|
const isSelfieMode = this.options.modelPath
|
|
2622
|
-
? this.options.modelPath
|
|
2631
|
+
? this.options.modelPath.includes('selfie_segmenter')
|
|
2623
2632
|
: true;
|
|
2624
2633
|
if (this.options.backgroundFilter === 'image') {
|
|
2634
|
+
const source = await this.loadBackground(this.options.backgroundImage);
|
|
2625
2635
|
return {
|
|
2626
|
-
backgroundSource:
|
|
2636
|
+
backgroundSource: source,
|
|
2627
2637
|
bgBlur: 0,
|
|
2628
2638
|
bgBlurRadius: 0,
|
|
2629
2639
|
isSelfieMode,
|
|
@@ -2638,26 +2648,337 @@ class VirtualBackground {
|
|
|
2638
2648
|
};
|
|
2639
2649
|
}
|
|
2640
2650
|
const numeric = blurLevel ?? 5;
|
|
2641
|
-
const bgBlur = Math.min(numeric * 3, 30);
|
|
2642
|
-
const bgBlurRadius = Math.min(numeric, 10);
|
|
2643
2651
|
return {
|
|
2644
2652
|
backgroundSource: undefined,
|
|
2645
|
-
bgBlur,
|
|
2646
|
-
bgBlurRadius,
|
|
2653
|
+
bgBlur: Math.min(numeric * 3, 30),
|
|
2654
|
+
bgBlurRadius: Math.min(numeric, 10),
|
|
2647
2655
|
isSelfieMode,
|
|
2648
2656
|
};
|
|
2649
2657
|
}
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
if (
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
+
async loadBackground(url) {
|
|
2659
|
+
if (!url)
|
|
2660
|
+
return null;
|
|
2661
|
+
const result = await fetch(url);
|
|
2662
|
+
if (!result.ok)
|
|
2663
|
+
return null;
|
|
2664
|
+
return {
|
|
2665
|
+
type: 'image',
|
|
2666
|
+
media: await createImageBitmap(await result.blob()),
|
|
2667
|
+
url,
|
|
2668
|
+
};
|
|
2669
|
+
}
|
|
2670
|
+
onFlush() {
|
|
2671
|
+
this.destroySegmenter();
|
|
2672
|
+
}
|
|
2673
|
+
onStop() {
|
|
2674
|
+
this.webGlRenderer?.close();
|
|
2675
|
+
this.destroySegmenter();
|
|
2676
|
+
}
|
|
2677
|
+
destroySegmenter() {
|
|
2678
|
+
this.segmenter?.close();
|
|
2679
|
+
this.segmenter = null;
|
|
2658
2680
|
this.isSegmenterReady = false;
|
|
2659
2681
|
}
|
|
2682
|
+
get processorName() {
|
|
2683
|
+
return 'background-processor';
|
|
2684
|
+
}
|
|
2685
|
+
}
|
|
2686
|
+
|
|
2687
|
+
/**
|
|
2688
|
+
* Simple WebGL renderer for full-screen Gaussian blur.
|
|
2689
|
+
* Uses a two-pass separable Gaussian blur (horizontal then vertical).
|
|
2690
|
+
* Optimized for moderation use cases by blurring at reduced resolution (15% scale)
|
|
2691
|
+
* and upscaling back to full resolution for output.
|
|
2692
|
+
*/
|
|
2693
|
+
class FullScreenBlurRenderer {
|
|
2694
|
+
constructor(canvas) {
|
|
2695
|
+
this.inputTexture = null;
|
|
2696
|
+
this.isRunning = false;
|
|
2697
|
+
this.targetWidth = 0;
|
|
2698
|
+
this.targetHeight = 0;
|
|
2699
|
+
this.weightCache = new Map();
|
|
2700
|
+
this.canvas = canvas;
|
|
2701
|
+
const gl = canvas.getContext('webgl2', {
|
|
2702
|
+
alpha: false,
|
|
2703
|
+
antialias: false,
|
|
2704
|
+
desynchronized: true,
|
|
2705
|
+
});
|
|
2706
|
+
if (!gl)
|
|
2707
|
+
throw new Error('WebGL2 not supported');
|
|
2708
|
+
this.gl = gl;
|
|
2709
|
+
const vertexShaderSource = `#version 300 es
|
|
2710
|
+
precision highp float;
|
|
2711
|
+
in vec2 a_position;
|
|
2712
|
+
in vec2 a_texCoord;
|
|
2713
|
+
out vec2 v_texCoord;
|
|
2714
|
+
void main() {
|
|
2715
|
+
v_texCoord = a_texCoord;
|
|
2716
|
+
gl_Position = vec4(a_position, 0.0, 1.0);
|
|
2717
|
+
}
|
|
2718
|
+
`;
|
|
2719
|
+
const fragmentShaderSource = `#version 300 es
|
|
2720
|
+
precision highp float;
|
|
2721
|
+
in vec2 v_texCoord;
|
|
2722
|
+
out vec4 outColor;
|
|
2723
|
+
uniform sampler2D u_image;
|
|
2724
|
+
uniform vec2 u_texelSize;
|
|
2725
|
+
uniform vec2 u_direction;
|
|
2726
|
+
uniform float u_weights[25];
|
|
2727
|
+
void main() {
|
|
2728
|
+
vec4 color = vec4(0.0);
|
|
2729
|
+
for (int i = -12; i <= 12; i++) {
|
|
2730
|
+
float w = u_weights[i + 12];
|
|
2731
|
+
if (w == 0.0) continue;
|
|
2732
|
+
vec2 offset = float(i) * u_direction * u_texelSize;
|
|
2733
|
+
color += w * texture(u_image, v_texCoord + offset);
|
|
2734
|
+
}
|
|
2735
|
+
outColor = color;
|
|
2736
|
+
}
|
|
2737
|
+
`;
|
|
2738
|
+
this.blurProgramHandle = this.createAndLinkProgram(vertexShaderSource, fragmentShaderSource);
|
|
2739
|
+
const passthroughFragmentShaderSource = `#version 300 es
|
|
2740
|
+
precision highp float;
|
|
2741
|
+
in vec2 v_texCoord;
|
|
2742
|
+
out vec4 outColor;
|
|
2743
|
+
uniform sampler2D u_image;
|
|
2744
|
+
void main() {
|
|
2745
|
+
outColor = texture(u_image, v_texCoord);
|
|
2746
|
+
}
|
|
2747
|
+
`;
|
|
2748
|
+
this.passthroughProgramHandle = this.createAndLinkProgram(vertexShaderSource, passthroughFragmentShaderSource);
|
|
2749
|
+
const blurProgram = this.blurProgramHandle;
|
|
2750
|
+
const passthroughProgram = this.passthroughProgramHandle;
|
|
2751
|
+
this.blurLocations = {
|
|
2752
|
+
positionLocation: gl.getAttribLocation(blurProgram, 'a_position'),
|
|
2753
|
+
texCoordLocation: gl.getAttribLocation(blurProgram, 'a_texCoord'),
|
|
2754
|
+
imageLocation: gl.getUniformLocation(blurProgram, 'u_image'),
|
|
2755
|
+
texelSizeLocation: gl.getUniformLocation(blurProgram, 'u_texelSize'),
|
|
2756
|
+
directionLocation: gl.getUniformLocation(blurProgram, 'u_direction'),
|
|
2757
|
+
weightsLocation: gl.getUniformLocation(blurProgram, 'u_weights'),
|
|
2758
|
+
};
|
|
2759
|
+
this.passthroughLocations = {
|
|
2760
|
+
positionLocation: gl.getAttribLocation(passthroughProgram, 'a_position'),
|
|
2761
|
+
texCoordLocation: gl.getAttribLocation(passthroughProgram, 'a_texCoord'),
|
|
2762
|
+
imageLocation: gl.getUniformLocation(passthroughProgram, 'u_image'),
|
|
2763
|
+
};
|
|
2764
|
+
this.positionBuffer = gl.createBuffer();
|
|
2765
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2766
|
+
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
|
|
2767
|
+
this.texCoordBuffer = gl.createBuffer();
|
|
2768
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2769
|
+
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0]), gl.STATIC_DRAW);
|
|
2770
|
+
const createTexture2D = () => {
|
|
2771
|
+
const tex = gl.createTexture();
|
|
2772
|
+
if (!tex)
|
|
2773
|
+
throw new Error('Failed to create texture');
|
|
2774
|
+
gl.bindTexture(gl.TEXTURE_2D, tex);
|
|
2775
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
|
2776
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
|
2777
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
|
2778
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
|
2779
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2780
|
+
return tex;
|
|
2781
|
+
};
|
|
2782
|
+
const createFramebufferForTexture = (tex) => {
|
|
2783
|
+
const fb = gl.createFramebuffer();
|
|
2784
|
+
if (!fb)
|
|
2785
|
+
throw new Error('Failed to create framebuffer');
|
|
2786
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
|
|
2787
|
+
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
|
|
2788
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2789
|
+
return fb;
|
|
2790
|
+
};
|
|
2791
|
+
this.pingTexture = createTexture2D();
|
|
2792
|
+
this.pongTexture = createTexture2D();
|
|
2793
|
+
this.pingFbo = createFramebufferForTexture(this.pingTexture);
|
|
2794
|
+
this.pongFbo = createFramebufferForTexture(this.pongTexture);
|
|
2795
|
+
this.inputTexture = createTexture2D();
|
|
2796
|
+
gl.useProgram(blurProgram);
|
|
2797
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2798
|
+
gl.enableVertexAttribArray(this.blurLocations.positionLocation);
|
|
2799
|
+
gl.vertexAttribPointer(this.blurLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2800
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2801
|
+
gl.enableVertexAttribArray(this.blurLocations.texCoordLocation);
|
|
2802
|
+
gl.vertexAttribPointer(this.blurLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2803
|
+
if (this.blurLocations.imageLocation) {
|
|
2804
|
+
gl.uniform1i(this.blurLocations.imageLocation, 0);
|
|
2805
|
+
}
|
|
2806
|
+
this.isRunning = true;
|
|
2807
|
+
}
|
|
2808
|
+
createAndLinkProgram(vsSource, fsSource) {
|
|
2809
|
+
const gl = this.gl;
|
|
2810
|
+
const vs = this.createShader(gl.VERTEX_SHADER, vsSource);
|
|
2811
|
+
const fs = this.createShader(gl.FRAGMENT_SHADER, fsSource);
|
|
2812
|
+
const prog = gl.createProgram();
|
|
2813
|
+
if (!prog)
|
|
2814
|
+
throw new Error('Failed to create program');
|
|
2815
|
+
gl.attachShader(prog, vs);
|
|
2816
|
+
gl.attachShader(prog, fs);
|
|
2817
|
+
gl.linkProgram(prog);
|
|
2818
|
+
if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) {
|
|
2819
|
+
throw new Error('Shader link failed: ' + gl.getProgramInfoLog(prog));
|
|
2820
|
+
}
|
|
2821
|
+
gl.deleteShader(vs);
|
|
2822
|
+
gl.deleteShader(fs);
|
|
2823
|
+
return prog;
|
|
2824
|
+
}
|
|
2825
|
+
createShader(type, source) {
|
|
2826
|
+
const gl = this.gl;
|
|
2827
|
+
const shader = gl.createShader(type);
|
|
2828
|
+
if (!shader)
|
|
2829
|
+
throw new Error('Failed to create shader');
|
|
2830
|
+
gl.shaderSource(shader, source);
|
|
2831
|
+
gl.compileShader(shader);
|
|
2832
|
+
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
|
2833
|
+
throw new Error('Shader compile failed: ' + gl.getShaderInfoLog(shader));
|
|
2834
|
+
}
|
|
2835
|
+
return shader;
|
|
2836
|
+
}
|
|
2837
|
+
getGaussianWeights(radius) {
|
|
2838
|
+
const r = Math.max(0, Math.min(radius | 0, 12));
|
|
2839
|
+
const cached = this.weightCache.get(r);
|
|
2840
|
+
if (cached)
|
|
2841
|
+
return cached;
|
|
2842
|
+
const weights = new Float32Array(25);
|
|
2843
|
+
if (r === 0) {
|
|
2844
|
+
weights[12] = 1.0;
|
|
2845
|
+
this.weightCache.set(r, weights);
|
|
2846
|
+
return weights;
|
|
2847
|
+
}
|
|
2848
|
+
const sigma = r * 0.6;
|
|
2849
|
+
let sum = 0;
|
|
2850
|
+
for (let i = -r; i <= r; i++) {
|
|
2851
|
+
const w = Math.exp(-(i * i) / (2 * sigma * sigma));
|
|
2852
|
+
weights[i + 12] = w;
|
|
2853
|
+
sum += w;
|
|
2854
|
+
}
|
|
2855
|
+
for (let i = -r; i <= r; i++) {
|
|
2856
|
+
weights[i + 12] /= sum;
|
|
2857
|
+
}
|
|
2858
|
+
this.weightCache.set(r, weights);
|
|
2859
|
+
return weights;
|
|
2860
|
+
}
|
|
2861
|
+
render(frame, radius) {
|
|
2862
|
+
if (!this.isRunning)
|
|
2863
|
+
return;
|
|
2864
|
+
const gl = this.gl;
|
|
2865
|
+
const width = frame.displayWidth;
|
|
2866
|
+
const height = frame.displayHeight;
|
|
2867
|
+
if (!width || !height)
|
|
2868
|
+
return;
|
|
2869
|
+
if (this.canvas.width !== width || this.canvas.height !== height) {
|
|
2870
|
+
this.canvas.width = width;
|
|
2871
|
+
this.canvas.height = height;
|
|
2872
|
+
}
|
|
2873
|
+
const scale = 0.15;
|
|
2874
|
+
const scaledWidth = Math.max(1, Math.floor(width * scale));
|
|
2875
|
+
const scaledHeight = Math.max(1, Math.floor(height * scale));
|
|
2876
|
+
if (scaledWidth !== this.targetWidth ||
|
|
2877
|
+
scaledHeight !== this.targetHeight) {
|
|
2878
|
+
this.targetWidth = scaledWidth;
|
|
2879
|
+
this.targetHeight = scaledHeight;
|
|
2880
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
|
|
2881
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2882
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
|
|
2883
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, scaledWidth, scaledHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
|
2884
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
2885
|
+
}
|
|
2886
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
2887
|
+
gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
|
|
2888
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
|
|
2889
|
+
gl.useProgram(this.blurProgramHandle);
|
|
2890
|
+
if (this.blurLocations.texelSizeLocation) {
|
|
2891
|
+
gl.uniform2f(this.blurLocations.texelSizeLocation, 1.0 / scaledWidth, 1.0 / scaledHeight);
|
|
2892
|
+
}
|
|
2893
|
+
const weights = this.getGaussianWeights(radius);
|
|
2894
|
+
if (this.blurLocations.weightsLocation) {
|
|
2895
|
+
gl.uniform1fv(this.blurLocations.weightsLocation, weights);
|
|
2896
|
+
}
|
|
2897
|
+
gl.viewport(0, 0, scaledWidth, scaledHeight);
|
|
2898
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, this.pingFbo);
|
|
2899
|
+
gl.bindTexture(gl.TEXTURE_2D, this.inputTexture);
|
|
2900
|
+
if (this.blurLocations.directionLocation) {
|
|
2901
|
+
gl.uniform2f(this.blurLocations.directionLocation, 1.0, 0.0);
|
|
2902
|
+
}
|
|
2903
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2904
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, this.pongFbo);
|
|
2905
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pingTexture);
|
|
2906
|
+
if (this.blurLocations.directionLocation) {
|
|
2907
|
+
gl.uniform2f(this.blurLocations.directionLocation, 0.0, 1.0);
|
|
2908
|
+
}
|
|
2909
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2910
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
2911
|
+
gl.viewport(0, 0, width, height);
|
|
2912
|
+
gl.useProgram(this.passthroughProgramHandle);
|
|
2913
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer);
|
|
2914
|
+
gl.enableVertexAttribArray(this.passthroughLocations.positionLocation);
|
|
2915
|
+
gl.vertexAttribPointer(this.passthroughLocations.positionLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2916
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer);
|
|
2917
|
+
gl.enableVertexAttribArray(this.passthroughLocations.texCoordLocation);
|
|
2918
|
+
gl.vertexAttribPointer(this.passthroughLocations.texCoordLocation, 2, gl.FLOAT, false, 0, 0);
|
|
2919
|
+
gl.bindTexture(gl.TEXTURE_2D, this.pongTexture);
|
|
2920
|
+
if (this.passthroughLocations.imageLocation) {
|
|
2921
|
+
gl.uniform1i(this.passthroughLocations.imageLocation, 0);
|
|
2922
|
+
}
|
|
2923
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
2924
|
+
}
|
|
2925
|
+
close() {
|
|
2926
|
+
if (!this.isRunning)
|
|
2927
|
+
return;
|
|
2928
|
+
this.isRunning = false;
|
|
2929
|
+
const gl = this.gl;
|
|
2930
|
+
if (this.pingFbo)
|
|
2931
|
+
gl.deleteFramebuffer(this.pingFbo);
|
|
2932
|
+
if (this.pongFbo)
|
|
2933
|
+
gl.deleteFramebuffer(this.pongFbo);
|
|
2934
|
+
if (this.pingTexture)
|
|
2935
|
+
gl.deleteTexture(this.pingTexture);
|
|
2936
|
+
if (this.pongTexture)
|
|
2937
|
+
gl.deleteTexture(this.pongTexture);
|
|
2938
|
+
if (this.inputTexture)
|
|
2939
|
+
gl.deleteTexture(this.inputTexture);
|
|
2940
|
+
if (this.positionBuffer)
|
|
2941
|
+
gl.deleteBuffer(this.positionBuffer);
|
|
2942
|
+
if (this.texCoordBuffer)
|
|
2943
|
+
gl.deleteBuffer(this.texCoordBuffer);
|
|
2944
|
+
gl.deleteProgram(this.blurProgramHandle);
|
|
2945
|
+
gl.deleteProgram(this.passthroughProgramHandle);
|
|
2946
|
+
}
|
|
2947
|
+
}
|
|
2948
|
+
|
|
2949
|
+
/**
|
|
2950
|
+
* A video filter that applies a full-screen blur to each frame.
|
|
2951
|
+
*
|
|
2952
|
+
* It uses a WebGL renderer to blur the incoming camera track and outputs
|
|
2953
|
+
* a new track with the effect applied. Setup and frame handling are managed
|
|
2954
|
+
* by the base processor.
|
|
2955
|
+
*/
|
|
2956
|
+
class FullScreenBlur extends BaseVideoProcessor {
|
|
2957
|
+
/**
|
|
2958
|
+
* Creates a new full-screen blur processor for the given video track.
|
|
2959
|
+
*
|
|
2960
|
+
* @param track - The input camera track to blur.
|
|
2961
|
+
* @param options - Optional settings such as the blur radius.
|
|
2962
|
+
* @param hooks - Optional callbacks for stats and error reporting.
|
|
2963
|
+
*/
|
|
2964
|
+
constructor(track, options = {}, hooks = {}) {
|
|
2965
|
+
super(track, hooks);
|
|
2966
|
+
this.blurRadius = options.blurRadius ?? 6;
|
|
2967
|
+
}
|
|
2968
|
+
async initialize() {
|
|
2969
|
+
this.blurRenderer = new FullScreenBlurRenderer(this.canvas);
|
|
2970
|
+
}
|
|
2971
|
+
async transform(frame) {
|
|
2972
|
+
this.blurRenderer.render(frame, this.blurRadius);
|
|
2973
|
+
return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
|
|
2974
|
+
}
|
|
2975
|
+
onStop() {
|
|
2976
|
+
this.blurRenderer?.close();
|
|
2977
|
+
}
|
|
2978
|
+
get processorName() {
|
|
2979
|
+
return 'fullscreen-blur';
|
|
2980
|
+
}
|
|
2660
2981
|
}
|
|
2661
2982
|
|
|
2662
|
-
export { BACKGROUND_BLUR_MAP, SegmentationLevel, VirtualBackground, createRenderer, isMediaPipePlatformSupported, isPlatformSupported, loadMediaPipe, loadTFLite };
|
|
2983
|
+
export { BACKGROUND_BLUR_MAP, FullScreenBlur, SegmentationLevel, VirtualBackground, createRenderer, isMediaPipePlatformSupported, isPlatformSupported, loadMediaPipe, loadTFLite };
|
|
2663
2984
|
//# sourceMappingURL=index.es.js.map
|