@stream-io/video-react-sdk 1.42.0 → 1.43.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -6
- package/dist/{background-filters-folwWrgs.cjs.js → background-filters-DMStbSEg.cjs.js} +97 -71
- package/dist/background-filters-DMStbSEg.cjs.js.map +1 -0
- package/dist/{embedded-BackgroundFilters-folwWrgs.cjs.js → embedded-BackgroundFilters-DMStbSEg.cjs.js} +97 -71
- package/dist/embedded-BackgroundFilters-DMStbSEg.cjs.js.map +1 -0
- package/dist/embedded.cjs.js +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/src/components/BackgroundFilters/BackgroundFilters.js +96 -70
- package/dist/esm/src/components/BackgroundFilters/BackgroundFilters.js.map +1 -1
- package/dist/esm/src/components/DeviceSettings/SpeakerTest.js +1 -1
- package/dist/index.cjs.js +3 -3
- package/package.json +4 -4
- package/src/components/BackgroundFilters/BackgroundFilters.tsx +139 -103
- package/dist/background-filters-folwWrgs.cjs.js.map +0 -1
- package/dist/embedded-BackgroundFilters-folwWrgs.cjs.js.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,17 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
This file was generated using [@jscutlery/semver](https://github.com/jscutlery/semver).
|
|
4
4
|
|
|
5
|
+
## [1.43.1](https://github.com/GetStream/stream-video-js/compare/@stream-io/video-react-sdk-1.43.0...@stream-io/video-react-sdk-1.43.1) (2026-09-11)
|
|
6
|
+
|
|
7
|
+
### Dependency Updates
|
|
8
|
+
|
|
9
|
+
- `@stream-io/video-client` updated to version `1.60.0`
|
|
10
|
+
- **client:** identify the SDK and version on SFU RPCs ([#2425](https://github.com/GetStream/stream-video-js/issues/2425)) ([107ff39](https://github.com/GetStream/stream-video-js/commit/107ff390c706beca4417d4ef5e2c7ec9bbbc321d))
|
|
11
|
+
- **client:** poll ring state to reconcile a dropped ring outcome ([#2393](https://github.com/GetStream/stream-video-js/issues/2393)) ([4605ac1](https://github.com/GetStream/stream-video-js/commit/4605ac1daf603382b7d3a6267e4852ac9fbfc0eb)), closes [#2422](https://github.com/GetStream/stream-video-js/issues/2422)
|
|
12
|
+
- `@stream-io/video-react-bindings` updated to version `1.20.3`
|
|
13
|
+
|
|
14
|
+
## [1.43.0](https://github.com/GetStream/stream-video-js/compare/@stream-io/video-react-sdk-1.42.0...@stream-io/video-react-sdk-1.43.0) (2026-09-04)
|
|
15
|
+
|
|
16
|
+
### Dependency Updates
|
|
17
|
+
|
|
18
|
+
- `@stream-io/video-client` updated to version `1.59.1`
|
|
19
|
+
- `@stream-io/video-filters-web` updated to version `0.10.0`
|
|
20
|
+
- `@stream-io/video-react-bindings` updated to version `1.20.2`
|
|
21
|
+
|
|
22
|
+
### Features
|
|
23
|
+
|
|
24
|
+
- **video-filters-web:** apply background filter changes without re-registering the filter ([#2403](https://github.com/GetStream/stream-video-js/issues/2403)) ([4fbb30a](https://github.com/GetStream/stream-video-js/commit/4fbb30a640f87a61d400179b24f844b04f2f3bd5))
|
|
25
|
+
|
|
5
26
|
## [1.42.0](https://github.com/GetStream/stream-video-js/compare/@stream-io/video-react-sdk-1.41.0...@stream-io/video-react-sdk-1.42.0) (2026-08-28)
|
|
6
27
|
|
|
7
28
|
### Dependency Updates
|
|
8
29
|
|
|
9
30
|
- `@stream-io/video-client` updated to version `1.59.0`
|
|
10
|
-
- do not end callingx service without call leave ([#2396](https://github.com/GetStream/stream-video-js/issues/2396)) ([15d6abb](https://github.com/GetStream/stream-video-js/commit/15d6abbfa9f4cc2a47c01d8fc8204016a2f8a448))
|
|
11
|
-
- **client:** keep devices disabled when querying calls ([#2382](https://github.com/GetStream/stream-video-js/issues/2382)) ([1c69755](https://github.com/GetStream/stream-video-js/commit/1c6975546dd70baf991410300fbf6de5ddfd9c65))
|
|
12
|
-
- **client:** Prevent concurrent joins ([#2386](https://github.com/GetStream/stream-video-js/issues/2386)) ([92b2cf6](https://github.com/GetStream/stream-video-js/commit/92b2cf62a719ffb98156250eeffa706841c6781c))
|
|
13
|
-
- **client:** refresh live call state on reconnect re-watch ([#2383](https://github.com/GetStream/stream-video-js/issues/2383)) ([d28067e](https://github.com/GetStream/stream-video-js/commit/d28067e543592e9ab8d8219fcaa9250d938dbc70))
|
|
14
|
-
- **client:** use generated call reporting types and remove unused fields ([#2377](https://github.com/GetStream/stream-video-js/issues/2377)) ([2eab662](https://github.com/GetStream/stream-video-js/commit/2eab66293c41c8e1d6467f2536525511186862e2))
|
|
15
|
-
- missing own capabilities for incoming call ([#2388](https://github.com/GetStream/stream-video-js/issues/2388)) ([0a04d34](https://github.com/GetStream/stream-video-js/commit/0a04d34a258d7ab3702348a1f929f87e62531875))
|
|
16
31
|
- `@stream-io/video-filters-web` updated to version `0.8.7`
|
|
17
32
|
- `@stream-io/video-react-bindings` updated to version `1.20.1`
|
|
18
33
|
|
|
@@ -276,8 +276,9 @@ const BackgroundFilters = (props) => {
|
|
|
276
276
|
handleErrorRef.current = onError;
|
|
277
277
|
const handleStatsRef = react.useRef(undefined);
|
|
278
278
|
handleStatsRef.current = onStats;
|
|
279
|
+
const filterActive = !!backgroundFilter;
|
|
279
280
|
react.useEffect(() => {
|
|
280
|
-
if (!call || !
|
|
281
|
+
if (!call || !filterActive)
|
|
281
282
|
return;
|
|
282
283
|
setIsLoading(true);
|
|
283
284
|
const { unregister, registered } = call.camera.registerFilter((ms) => {
|
|
@@ -289,11 +290,19 @@ const BackgroundFilters = (props) => {
|
|
|
289
290
|
return () => {
|
|
290
291
|
unregister().catch((err) => console.warn(`Can't unregister filter`, err));
|
|
291
292
|
};
|
|
292
|
-
}, [call, start,
|
|
293
|
+
}, [call, start, filterActive, setIsLoading]);
|
|
293
294
|
return children;
|
|
294
295
|
};
|
|
295
296
|
const useRenderer = (api, tfLite, call, engine) => {
|
|
296
297
|
const { backgroundFilter, backgroundBlurLevel, backgroundImage, modelFilePath, basePath, segmentationOptions, } = api;
|
|
298
|
+
const optionsRef = react.useRef({});
|
|
299
|
+
optionsRef.current = {
|
|
300
|
+
backgroundFilter,
|
|
301
|
+
backgroundBlurLevel,
|
|
302
|
+
backgroundImage,
|
|
303
|
+
segmentationOptions,
|
|
304
|
+
};
|
|
305
|
+
const processorRef = react.useRef(undefined);
|
|
297
306
|
const videoRef = react.useRef(null);
|
|
298
307
|
const canvasRef = react.useRef(null);
|
|
299
308
|
const bgImageRef = react.useRef(null);
|
|
@@ -301,9 +310,53 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
301
310
|
width: 1920,
|
|
302
311
|
height: 1080,
|
|
303
312
|
});
|
|
304
|
-
const
|
|
313
|
+
const startMediaPipe = react.useCallback((ms, onError, onStats) => {
|
|
305
314
|
let outputStream;
|
|
306
315
|
let processor;
|
|
316
|
+
const output = new Promise((resolve, reject) => {
|
|
317
|
+
const options = optionsRef.current;
|
|
318
|
+
if (!options.backgroundFilter) {
|
|
319
|
+
reject(new Error('No filter specified'));
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
const [track] = ms.getVideoTracks();
|
|
323
|
+
if (!track) {
|
|
324
|
+
reject(new Error('No video tracks in input media stream'));
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
call?.tracer.trace('backgroundFilters.enable', {
|
|
328
|
+
...options,
|
|
329
|
+
engine: FilterEngine.MEDIA_PIPE,
|
|
330
|
+
modelFilePath,
|
|
331
|
+
});
|
|
332
|
+
const trackSettings = track.getSettings();
|
|
333
|
+
reactDom.flushSync(() => setVideoSize({
|
|
334
|
+
width: trackSettings.width ?? 0,
|
|
335
|
+
height: trackSettings.height ?? 0,
|
|
336
|
+
}));
|
|
337
|
+
processor = new videoFiltersWeb.VirtualBackground(track, { ...options, basePath, modelPath: modelFilePath }, { onError, onStats });
|
|
338
|
+
processorRef.current = processor;
|
|
339
|
+
processor
|
|
340
|
+
.start()
|
|
341
|
+
.then((processedTrack) => {
|
|
342
|
+
outputStream = new MediaStream([processedTrack]);
|
|
343
|
+
resolve(outputStream);
|
|
344
|
+
})
|
|
345
|
+
.catch(reject);
|
|
346
|
+
});
|
|
347
|
+
return {
|
|
348
|
+
output,
|
|
349
|
+
stop: () => {
|
|
350
|
+
call?.tracer.trace('backgroundFilters.disable', null);
|
|
351
|
+
processorRef.current = undefined;
|
|
352
|
+
processor?.stop();
|
|
353
|
+
if (outputStream)
|
|
354
|
+
videoClient.disposeOfMediaStream(outputStream);
|
|
355
|
+
},
|
|
356
|
+
};
|
|
357
|
+
}, [call?.tracer, modelFilePath, basePath]);
|
|
358
|
+
const startTf = react.useCallback((ms, onError) => {
|
|
359
|
+
let outputStream;
|
|
307
360
|
let renderer;
|
|
308
361
|
const output = new Promise((resolve, reject) => {
|
|
309
362
|
if (!backgroundFilter) {
|
|
@@ -318,83 +371,42 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
318
371
|
reject(new Error('No video tracks in input media stream'));
|
|
319
372
|
return;
|
|
320
373
|
}
|
|
321
|
-
if (
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
modelFilePath,
|
|
328
|
-
});
|
|
329
|
-
if (!videoEl) {
|
|
330
|
-
reject(new Error('Renderer started before elements are ready'));
|
|
331
|
-
return;
|
|
332
|
-
}
|
|
374
|
+
if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
|
|
375
|
+
reject(new Error('Renderer started before elements are ready'));
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
videoEl.srcObject = ms;
|
|
379
|
+
videoEl.play().then(() => {
|
|
333
380
|
const trackSettings = track.getSettings();
|
|
334
381
|
reactDom.flushSync(() => setVideoSize({
|
|
335
382
|
width: trackSettings.width ?? 0,
|
|
336
383
|
height: trackSettings.height ?? 0,
|
|
337
384
|
}));
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
modelPath: modelFilePath,
|
|
385
|
+
call?.tracer.trace('backgroundFilters.enable', {
|
|
386
|
+
backgroundFilter,
|
|
341
387
|
backgroundBlurLevel,
|
|
342
388
|
backgroundImage,
|
|
343
|
-
|
|
344
|
-
segmentationOptions,
|
|
345
|
-
}, { onError, onStats });
|
|
346
|
-
processor
|
|
347
|
-
.start()
|
|
348
|
-
.then((processedTrack) => {
|
|
349
|
-
outputStream = new MediaStream([processedTrack]);
|
|
350
|
-
resolve(outputStream);
|
|
351
|
-
})
|
|
352
|
-
.catch((error) => {
|
|
353
|
-
reject(error);
|
|
389
|
+
engine: FilterEngine.TF,
|
|
354
390
|
});
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
if (engine === FilterEngine.TF) {
|
|
358
|
-
if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
|
|
359
|
-
reject(new Error('Renderer started before elements are ready'));
|
|
391
|
+
if (!tfLite) {
|
|
392
|
+
reject(new Error('TensorFlow Lite not loaded'));
|
|
360
393
|
return;
|
|
361
394
|
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
backgroundImage,
|
|
373
|
-
engine,
|
|
374
|
-
});
|
|
375
|
-
if (!tfLite) {
|
|
376
|
-
reject(new Error('TensorFlow Lite not loaded'));
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
renderer = videoFiltersWeb.createRenderer(tfLite, videoEl, canvasEl, {
|
|
380
|
-
backgroundFilter,
|
|
381
|
-
backgroundBlurLevel,
|
|
382
|
-
backgroundImage: bgImageEl ?? undefined,
|
|
383
|
-
}, onError);
|
|
384
|
-
outputStream = canvasEl.captureStream();
|
|
385
|
-
resolve(outputStream);
|
|
386
|
-
}, () => {
|
|
387
|
-
reject(new Error('Could not play the source video stream'));
|
|
388
|
-
});
|
|
389
|
-
return;
|
|
390
|
-
}
|
|
391
|
-
reject(new Error('No supported engine available'));
|
|
395
|
+
renderer = videoFiltersWeb.createRenderer(tfLite, videoEl, canvasEl, {
|
|
396
|
+
backgroundFilter,
|
|
397
|
+
backgroundBlurLevel,
|
|
398
|
+
backgroundImage: bgImageEl ?? undefined,
|
|
399
|
+
}, onError);
|
|
400
|
+
outputStream = canvasEl.captureStream();
|
|
401
|
+
resolve(outputStream);
|
|
402
|
+
}, () => {
|
|
403
|
+
reject(new Error('Could not play the source video stream'));
|
|
404
|
+
});
|
|
392
405
|
});
|
|
393
406
|
return {
|
|
394
407
|
output,
|
|
395
408
|
stop: () => {
|
|
396
409
|
call?.tracer.trace('backgroundFilters.disable', null);
|
|
397
|
-
processor?.stop();
|
|
398
410
|
renderer?.dispose();
|
|
399
411
|
if (videoRef.current)
|
|
400
412
|
videoRef.current.srcObject = null;
|
|
@@ -403,14 +415,28 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
403
415
|
},
|
|
404
416
|
};
|
|
405
417
|
}, [
|
|
418
|
+
call?.tracer,
|
|
419
|
+
tfLite,
|
|
420
|
+
backgroundFilter,
|
|
406
421
|
backgroundBlurLevel,
|
|
422
|
+
backgroundImage,
|
|
423
|
+
]);
|
|
424
|
+
const start = engine === FilterEngine.TF ? startTf : startMediaPipe;
|
|
425
|
+
react.useEffect(() => {
|
|
426
|
+
if (!backgroundFilter)
|
|
427
|
+
return;
|
|
428
|
+
processorRef.current
|
|
429
|
+
?.updateOptions({
|
|
430
|
+
backgroundFilter,
|
|
431
|
+
backgroundBlurLevel,
|
|
432
|
+
backgroundImage,
|
|
433
|
+
segmentationOptions,
|
|
434
|
+
})
|
|
435
|
+
.catch((err) => console.warn(`[filters] Can't apply the background options`, err));
|
|
436
|
+
}, [
|
|
407
437
|
backgroundFilter,
|
|
438
|
+
backgroundBlurLevel,
|
|
408
439
|
backgroundImage,
|
|
409
|
-
call?.tracer,
|
|
410
|
-
tfLite,
|
|
411
|
-
engine,
|
|
412
|
-
modelFilePath,
|
|
413
|
-
basePath,
|
|
414
440
|
segmentationOptions,
|
|
415
441
|
]);
|
|
416
442
|
const children = (jsxRuntime.jsxs("div", { className: "str-video__background-filters", children: [jsxRuntime.jsx("video", { className: clsx('str-video__background-filters__video', videoSize.height > videoSize.width &&
|
|
@@ -419,4 +445,4 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
419
445
|
};
|
|
420
446
|
|
|
421
447
|
exports.BackgroundFiltersProvider = BackgroundFiltersProvider;
|
|
422
|
-
//# sourceMappingURL=background-filters-
|
|
448
|
+
//# sourceMappingURL=background-filters-DMStbSEg.cjs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"background-filters-DMStbSEg.cjs.js","sources":["../../src/components/BackgroundFilters/BackgroundFilters.tsx"],"sourcesContent":["import {\n Context,\n PropsWithChildren,\n useCallback,\n useEffect,\n useMemo,\n useRef,\n useState,\n} from 'react';\nimport { flushSync } from 'react-dom';\nimport { useCall, useCallStateHooks } from '@stream-io/video-react-bindings';\nimport { Call, disposeOfMediaStream } from '@stream-io/video-client';\nimport {\n BackgroundBlurLevel,\n BackgroundEffectOptions,\n createRenderer,\n isMediaPipePlatformSupported,\n isPlatformSupported,\n loadMediaPipe,\n loadTFLite,\n PerformanceStats,\n Renderer,\n TFLite,\n VirtualBackground,\n} from '@stream-io/video-filters-web';\nimport clsx from 'clsx';\nimport type {\n BackgroundFiltersPerformance,\n BackgroundFiltersProps,\n BackgroundFiltersContextValue,\n PerformanceDegradationReason,\n} from './types';\n\ntype MediaStreamTrackVideoStats = {\n deliveredFrames: number;\n discardedFrames: number;\n totalFrames: number;\n};\n\ntype MediaStreamTrackWithStats = MediaStreamTrack & {\n stats?: MediaStreamTrackVideoStats;\n};\n\n/**\n * Constants for FPS warning calculation.\n * Smooths the processed-to-source FPS ratio and uses hysteresis\n * so the degradation warning doesn't flicker near the limit.\n */\nconst ALPHA = 0.2;\nconst FPS_RATIO_WARNING_THRESHOLD_LOWER = 0.75;\nconst FPS_RATIO_WARNING_THRESHOLD_UPPER = 0.85;\n\nconst EMPTY_BACKGROUND_IMAGES: string[] = [];\n\n/**\n * Represents the available background filter processing engines.\n */\nenum FilterEngine {\n TF,\n MEDIA_PIPE,\n NONE,\n}\n\n/**\n * Determines which filter engine is available.\n * MEDIA_PIPE is the default unless legacy filters are requested or MediaPipe is unsupported.\n *\n * Returns NONE if neither is supported.\n */\nconst determineEngine = async (\n useLegacyFilter: boolean | undefined,\n forceSafariSupport: boolean | undefined,\n forceMobileSupport: boolean | undefined,\n): Promise<FilterEngine> => {\n if (useLegacyFilter) {\n const isTfPlatformSupported = await isPlatformSupported({\n forceSafariSupport,\n forceMobileSupport,\n });\n return isTfPlatformSupported ? FilterEngine.TF : FilterEngine.NONE;\n }\n\n const isMediaPipeSupported = await isMediaPipePlatformSupported({\n forceSafariSupport,\n forceMobileSupport,\n });\n\n return isMediaPipeSupported ? FilterEngine.MEDIA_PIPE : FilterEngine.NONE;\n};\n\n/**\n * Samples the raw camera track's frame stats (W3C MediaCapture Extensions) at a\n * fixed interval and returns a live source fps estimate. Returns `undefined`\n * while warming up, when the camera is off, or when the browser doesn't expose\n * `MediaStreamTrack.stats` (currently Chromium-only).\n */\nconst useTrackFramesPerSecond = (call: Call | undefined) => {\n const { useCameraState } = useCallStateHooks();\n const { rootMediaStream } = useCameraState();\n const [fps, setFps] = useState<number | undefined>(undefined);\n const trackId = rootMediaStream?.getVideoTracks()[0]?.id;\n\n useEffect(() => {\n if (!call) {\n setFps(undefined);\n return;\n }\n\n let previousSnapshot:\n (MediaStreamTrackVideoStats & { capturedAt: number }) | undefined;\n let previousTrackId: string | undefined;\n\n const intervalId = setInterval(() => {\n const track = rootMediaStream?.getVideoTracks()[0] as\n MediaStreamTrackWithStats | undefined;\n\n const stats = track?.stats;\n if (!track || !stats) {\n previousSnapshot = undefined;\n setFps(undefined);\n return;\n }\n\n if (track.id !== previousTrackId) {\n previousTrackId = track.id;\n previousSnapshot = undefined;\n setFps(undefined);\n }\n\n const currentSnapshot = {\n deliveredFrames: stats.deliveredFrames,\n discardedFrames: stats.discardedFrames,\n totalFrames: stats.totalFrames,\n capturedAt: performance.now(),\n };\n\n if (previousSnapshot) {\n const elapsedSec =\n (currentSnapshot.capturedAt - previousSnapshot.capturedAt) / 1000;\n\n const delivered =\n currentSnapshot.deliveredFrames - previousSnapshot.deliveredFrames;\n\n const nextFps = Math.round(delivered / elapsedSec);\n setFps(nextFps);\n }\n previousSnapshot = currentSnapshot;\n }, 1000);\n\n return () => {\n clearInterval(intervalId);\n setFps(undefined);\n };\n }, [call, rootMediaStream, trackId]);\n\n return fps;\n};\n\n/**\n * A provider component that enables the use of background filters in your app.\n *\n * Please make sure you have the `@stream-io/video-filters-web` package installed\n * in your project before using this component.\n */\nexport const BackgroundFiltersProvider = (\n props: PropsWithChildren<BackgroundFiltersProps> & {\n // for code splitting. Prevents circular dependency issues where\n // this Context needs to be present in the main chunk, but also\n // imported by the background filters chunk.\n ContextProvider: Context<BackgroundFiltersContextValue | undefined>;\n },\n) => {\n const {\n ContextProvider,\n children,\n backgroundImages = EMPTY_BACKGROUND_IMAGES,\n backgroundFilter: bgFilterFromProps = undefined,\n backgroundImage: bgImageFromProps = undefined,\n backgroundBlurLevel: bgBlurLevelFromProps = undefined,\n tfFilePath,\n modelFilePath,\n useLegacyFilter,\n basePath,\n onError,\n forceSafariSupport,\n forceMobileSupport,\n segmentationOptions,\n } = props;\n\n const call = useCall();\n const { useCallStatsReport } = useCallStateHooks();\n const callStatsReport = useCallStatsReport();\n\n const [backgroundFilter, setBackgroundFilter] = useState(bgFilterFromProps);\n const [backgroundImage, setBackgroundImage] = useState(bgImageFromProps);\n const [backgroundBlurLevel, setBackgroundBlurLevel] =\n useState(bgBlurLevelFromProps);\n\n const [showLowFpsWarning, setShowLowFpsWarning] = useState<boolean>(false);\n const [isLoading, setIsLoading] = useState<boolean>(false);\n\n const rootFps = useTrackFramesPerSecond(call);\n const cameraFrameRate = callStatsReport?.publisherStats?.camera?.frameRate;\n\n const sourceFps = rootFps ?? cameraFrameRate;\n const fpsRatioEmaRef = useRef<number | undefined>(undefined);\n\n const handleStats = useCallback(\n (stats: PerformanceStats) => {\n const fps = stats?.fps;\n if (fps == null || !sourceFps) {\n fpsRatioEmaRef.current = undefined;\n setShowLowFpsWarning(false);\n return;\n }\n\n const ratio = fps / sourceFps;\n const prevRatioEma = fpsRatioEmaRef.current ?? ratio;\n\n const nextFpsRatioEma = ALPHA * ratio + (1 - ALPHA) * prevRatioEma;\n fpsRatioEmaRef.current = nextFpsRatioEma;\n\n setShowLowFpsWarning((prev) => {\n if (prev && nextFpsRatioEma > FPS_RATIO_WARNING_THRESHOLD_UPPER)\n return false;\n if (!prev && nextFpsRatioEma < FPS_RATIO_WARNING_THRESHOLD_LOWER)\n return true;\n\n return prev;\n });\n },\n [sourceFps],\n );\n\n const performance: BackgroundFiltersPerformance = useMemo(() => {\n if (!backgroundFilter) {\n return { degraded: false };\n }\n\n const reasons: Array<PerformanceDegradationReason> = [];\n\n if (showLowFpsWarning) {\n reasons.push('frame-drop');\n }\n\n const qualityLimitationReasons =\n callStatsReport?.publisherStats?.qualityLimitationReasons;\n\n if (\n showLowFpsWarning &&\n qualityLimitationReasons &&\n qualityLimitationReasons?.includes('cpu')\n ) {\n reasons.push('cpu-throttling');\n }\n\n return {\n degraded: reasons.length > 0,\n reason: reasons.length > 0 ? reasons : undefined,\n };\n }, [\n showLowFpsWarning,\n callStatsReport?.publisherStats?.qualityLimitationReasons,\n backgroundFilter,\n ]);\n\n const prevDegradedRef = useRef<boolean | undefined>(undefined);\n useEffect(() => {\n const currentDegraded = performance.degraded;\n const prevDegraded = prevDegradedRef.current;\n\n if (\n !!backgroundFilter &&\n prevDegraded !== undefined &&\n prevDegraded !== currentDegraded\n ) {\n call?.tracer.trace('backgroundFilters.performance', {\n degraded: currentDegraded,\n reason: performance?.reason,\n ratio: fpsRatioEmaRef.current,\n sourceFps,\n });\n }\n prevDegradedRef.current = currentDegraded;\n }, [\n performance.degraded,\n performance.reason,\n backgroundFilter,\n call?.tracer,\n sourceFps,\n ]);\n\n const applyBackgroundImageFilter = useCallback((imageUrl: string) => {\n setBackgroundFilter('image');\n setBackgroundImage(imageUrl);\n }, []);\n\n const applyBackgroundBlurFilter = useCallback(\n (blurLevel: BackgroundBlurLevel = 'high') => {\n setBackgroundFilter('blur');\n setBackgroundBlurLevel(blurLevel);\n },\n [],\n );\n\n const disableBackgroundFilter = useCallback(() => {\n setBackgroundFilter(undefined);\n setBackgroundImage(undefined);\n setBackgroundBlurLevel(undefined);\n\n fpsRatioEmaRef.current = undefined;\n setShowLowFpsWarning(false);\n }, []);\n\n const [engine, setEngine] = useState<FilterEngine>(FilterEngine.NONE);\n const [isSupported, setIsSupported] = useState(false);\n useEffect(() => {\n determineEngine(\n useLegacyFilter,\n forceSafariSupport,\n forceMobileSupport,\n ).then((determinedEngine) => {\n setEngine(determinedEngine);\n setIsSupported(determinedEngine !== FilterEngine.NONE);\n });\n }, [forceMobileSupport, forceSafariSupport, useLegacyFilter]);\n\n const [tfLite, setTfLite] = useState<TFLite>();\n useEffect(() => {\n if (engine !== FilterEngine.TF) return;\n\n loadTFLite({ basePath, modelFilePath, tfFilePath })\n .then(setTfLite)\n .catch((err) => console.error('Failed to load TFLite', err));\n }, [basePath, engine, modelFilePath, tfFilePath]);\n\n const [mediaPipe, setMediaPipe] = useState<ArrayBuffer>();\n useEffect(() => {\n if (engine !== FilterEngine.MEDIA_PIPE) return;\n\n loadMediaPipe({\n basePath: basePath,\n modelPath: modelFilePath,\n })\n .then(setMediaPipe)\n .catch((err) => console.error('Failed to preload MediaPipe', err));\n }, [engine, modelFilePath, basePath]);\n\n const handleError = useCallback(\n (error: any) => {\n console.warn(\n '[filters] Filter encountered an error and will be disabled',\n );\n disableBackgroundFilter();\n onError?.(error);\n },\n [disableBackgroundFilter, onError],\n );\n\n const isReady = useLegacyFilter ? !!tfLite : !!mediaPipe;\n\n const contextValue = useMemo<BackgroundFiltersContextValue>(\n () => ({\n isSupported,\n performance,\n isReady,\n isLoading,\n backgroundImage,\n backgroundBlurLevel,\n backgroundFilter,\n disableBackgroundFilter,\n applyBackgroundBlurFilter,\n applyBackgroundImageFilter,\n backgroundImages,\n tfFilePath,\n modelFilePath,\n basePath,\n onError: handleError,\n segmentationOptions,\n }),\n [\n isSupported,\n performance,\n isReady,\n isLoading,\n backgroundImage,\n backgroundBlurLevel,\n backgroundFilter,\n disableBackgroundFilter,\n applyBackgroundBlurFilter,\n applyBackgroundImageFilter,\n backgroundImages,\n tfFilePath,\n modelFilePath,\n basePath,\n handleError,\n segmentationOptions,\n ],\n );\n return (\n <ContextProvider.Provider value={contextValue}>\n {children}\n {isReady && (\n <BackgroundFilters\n api={contextValue}\n tfLite={tfLite}\n engine={engine}\n onStats={handleStats}\n setIsLoading={setIsLoading}\n />\n )}\n </ContextProvider.Provider>\n );\n};\n\nconst BackgroundFilters = (props: {\n api: BackgroundFiltersContextValue;\n tfLite?: TFLite;\n engine: FilterEngine;\n onStats: (stats: PerformanceStats) => void;\n setIsLoading: (loading: boolean) => void;\n}) => {\n const call = useCall();\n const { engine, api, tfLite, onStats, setIsLoading } = props;\n const { children, start } = useRenderer(api, tfLite, call, engine);\n const { onError, backgroundFilter } = api;\n const handleErrorRef = useRef<((error: any) => void) | undefined>(undefined);\n handleErrorRef.current = onError;\n\n const handleStatsRef = useRef<\n ((stats: PerformanceStats) => void) | undefined\n >(undefined);\n handleStatsRef.current = onStats;\n\n const filterActive = !!backgroundFilter;\n\n useEffect(() => {\n if (!call || !filterActive) return;\n\n setIsLoading(true);\n const { unregister, registered } = call.camera.registerFilter((ms) => {\n return start(\n ms,\n (error) => handleErrorRef.current?.(error),\n (stats: PerformanceStats) => handleStatsRef.current?.(stats),\n );\n });\n registered.finally(() => {\n setIsLoading(false);\n });\n\n return () => {\n unregister().catch((err) => console.warn(`Can't unregister filter`, err));\n };\n }, [call, start, filterActive, setIsLoading]);\n\n return children;\n};\n\nconst useRenderer = (\n api: BackgroundFiltersContextValue,\n tfLite: TFLite | undefined,\n call: Call | undefined,\n engine: FilterEngine,\n) => {\n const {\n backgroundFilter,\n backgroundBlurLevel,\n backgroundImage,\n modelFilePath,\n basePath,\n segmentationOptions,\n } = api;\n\n const optionsRef = useRef<BackgroundEffectOptions>({});\n optionsRef.current = {\n backgroundFilter,\n backgroundBlurLevel,\n backgroundImage,\n segmentationOptions,\n };\n\n const processorRef = useRef<VirtualBackground | undefined>(undefined);\n\n const videoRef = useRef<HTMLVideoElement>(null);\n const canvasRef = useRef<HTMLCanvasElement>(null);\n const bgImageRef = useRef<HTMLImageElement>(null);\n const [videoSize, setVideoSize] = useState<{ width: number; height: number }>(\n {\n width: 1920,\n height: 1080,\n },\n );\n\n const startMediaPipe = useCallback(\n (\n ms: MediaStream,\n onError?: (error: any) => void,\n onStats?: (stats: PerformanceStats) => void,\n ) => {\n let outputStream: MediaStream | undefined;\n let processor: VirtualBackground | undefined;\n\n const output = new Promise<MediaStream>((resolve, reject) => {\n const options = optionsRef.current;\n if (!options.backgroundFilter) {\n reject(new Error('No filter specified'));\n return;\n }\n\n const [track] = ms.getVideoTracks();\n if (!track) {\n reject(new Error('No video tracks in input media stream'));\n return;\n }\n\n call?.tracer.trace('backgroundFilters.enable', {\n ...options,\n engine: FilterEngine.MEDIA_PIPE,\n modelFilePath,\n });\n\n const trackSettings = track.getSettings();\n flushSync(() =>\n setVideoSize({\n width: trackSettings.width ?? 0,\n height: trackSettings.height ?? 0,\n }),\n );\n\n processor = new VirtualBackground(\n track,\n { ...options, basePath, modelPath: modelFilePath },\n { onError, onStats },\n );\n processorRef.current = processor;\n processor\n .start()\n .then((processedTrack) => {\n outputStream = new MediaStream([processedTrack]);\n resolve(outputStream);\n })\n .catch(reject);\n });\n\n return {\n output,\n stop: () => {\n call?.tracer.trace('backgroundFilters.disable', null);\n processorRef.current = undefined;\n processor?.stop();\n if (outputStream) disposeOfMediaStream(outputStream);\n },\n };\n },\n [call?.tracer, modelFilePath, basePath],\n );\n\n const startTf = useCallback(\n (ms: MediaStream, onError?: (error: any) => void) => {\n let outputStream: MediaStream | undefined;\n let renderer: Renderer | undefined;\n\n const output = new Promise<MediaStream>((resolve, reject) => {\n if (!backgroundFilter) {\n reject(new Error('No filter specified'));\n return;\n }\n\n const videoEl = videoRef.current;\n const canvasEl = canvasRef.current;\n const bgImageEl = bgImageRef.current;\n\n const [track] = ms.getVideoTracks();\n if (!track) {\n reject(new Error('No video tracks in input media stream'));\n return;\n }\n\n if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {\n reject(new Error('Renderer started before elements are ready'));\n return;\n }\n\n videoEl.srcObject = ms;\n videoEl.play().then(\n () => {\n const trackSettings = track.getSettings();\n flushSync(() =>\n setVideoSize({\n width: trackSettings.width ?? 0,\n height: trackSettings.height ?? 0,\n }),\n );\n call?.tracer.trace('backgroundFilters.enable', {\n backgroundFilter,\n backgroundBlurLevel,\n backgroundImage,\n engine: FilterEngine.TF,\n });\n\n if (!tfLite) {\n reject(new Error('TensorFlow Lite not loaded'));\n return;\n }\n\n renderer = createRenderer(\n tfLite,\n videoEl,\n canvasEl,\n {\n backgroundFilter,\n backgroundBlurLevel,\n backgroundImage: bgImageEl ?? undefined,\n },\n onError,\n );\n outputStream = canvasEl.captureStream();\n\n resolve(outputStream);\n },\n () => {\n reject(new Error('Could not play the source video stream'));\n },\n );\n });\n\n return {\n output,\n stop: () => {\n call?.tracer.trace('backgroundFilters.disable', null);\n renderer?.dispose();\n if (videoRef.current) videoRef.current.srcObject = null;\n if (outputStream) disposeOfMediaStream(outputStream);\n },\n };\n },\n [\n call?.tracer,\n tfLite,\n backgroundFilter,\n backgroundBlurLevel,\n backgroundImage,\n ],\n );\n\n const start = engine === FilterEngine.TF ? startTf : startMediaPipe;\n\n useEffect(() => {\n if (!backgroundFilter) return;\n\n processorRef.current\n ?.updateOptions({\n backgroundFilter,\n backgroundBlurLevel,\n backgroundImage,\n segmentationOptions,\n })\n .catch((err) =>\n console.warn(`[filters] Can't apply the background options`, err),\n );\n }, [\n backgroundFilter,\n backgroundBlurLevel,\n backgroundImage,\n segmentationOptions,\n ]);\n\n const children = (\n <div className=\"str-video__background-filters\">\n <video\n className={clsx(\n 'str-video__background-filters__video',\n videoSize.height > videoSize.width &&\n 'str-video__background-filters__video--tall',\n )}\n ref={videoRef}\n playsInline\n muted\n controls={false}\n {...videoSize}\n />\n {backgroundImage && (\n <img\n className=\"str-video__background-filters__background-image\"\n alt=\"Background\"\n ref={bgImageRef}\n crossOrigin=\"anonymous\"\n src={backgroundImage}\n {...videoSize}\n />\n )}\n <canvas\n className=\"str-video__background-filters__target-canvas\"\n {...videoSize}\n ref={canvasRef}\n />\n </div>\n );\n\n return { start, children };\n};\n"],"names":["isPlatformSupported","isMediaPipePlatformSupported","useCallStateHooks","useState","useEffect","useCall","useRef","useCallback","useMemo","loadTFLite","loadMediaPipe","_jsxs","_jsx","flushSync","VirtualBackground","disposeOfMediaStream","createRenderer"],"mappings":";;;;;;;;;;AA2CA;;;;AAIG;AACH,MAAM,KAAK,GAAG,GAAG;AACjB,MAAM,iCAAiC,GAAG,IAAI;AAC9C,MAAM,iCAAiC,GAAG,IAAI;AAE9C,MAAM,uBAAuB,GAAa,EAAE;AAE5C;;AAEG;AACH,IAAK,YAIJ;AAJD,CAAA,UAAK,YAAY,EAAA;AACf,IAAA,YAAA,CAAA,YAAA,CAAA,IAAA,CAAA,GAAA,CAAA,CAAA,GAAA,IAAE;AACF,IAAA,YAAA,CAAA,YAAA,CAAA,YAAA,CAAA,GAAA,CAAA,CAAA,GAAA,YAAU;AACV,IAAA,YAAA,CAAA,YAAA,CAAA,MAAA,CAAA,GAAA,CAAA,CAAA,GAAA,MAAI;AACN,CAAC,EAJI,YAAY,KAAZ,YAAY,GAAA,EAAA,CAAA,CAAA;AAMjB;;;;;AAKG;AACH,MAAM,eAAe,GAAG,OACtB,eAAoC,EACpC,kBAAuC,EACvC,kBAAuC,KACd;IACzB,IAAI,eAAe,EAAE;AACnB,QAAA,MAAM,qBAAqB,GAAG,MAAMA,mCAAmB,CAAC;YACtD,kBAAkB;YAClB,kBAAkB;AACnB,SAAA,CAAC;AACF,QAAA,OAAO,qBAAqB,GAAG,YAAY,CAAC,EAAE,GAAG,YAAY,CAAC,IAAI;IACpE;AAEA,IAAA,MAAM,oBAAoB,GAAG,MAAMC,4CAA4B,CAAC;QAC9D,kBAAkB;QAClB,kBAAkB;AACnB,KAAA,CAAC;AAEF,IAAA,OAAO,oBAAoB,GAAG,YAAY,CAAC,UAAU,GAAG,YAAY,CAAC,IAAI;AAC3E,CAAC;AAED;;;;;AAKG;AACH,MAAM,uBAAuB,GAAG,CAAC,IAAsB,KAAI;AACzD,IAAA,MAAM,EAAE,cAAc,EAAE,GAAGC,oCAAiB,EAAE;AAC9C,IAAA,MAAM,EAAE,eAAe,EAAE,GAAG,cAAc,EAAE;IAC5C,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAGC,cAAQ,CAAqB,SAAS,CAAC;IAC7D,MAAM,OAAO,GAAG,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;IAExDC,eAAS,CAAC,MAAK;QACb,IAAI,CAAC,IAAI,EAAE;YACT,MAAM,CAAC,SAAS,CAAC;YACjB;QACF;AAEA,QAAA,IAAI,gBAC+D;AACnE,QAAA,IAAI,eAAmC;AAEvC,QAAA,MAAM,UAAU,GAAG,WAAW,CAAC,MAAK;YAClC,MAAM,KAAK,GAAG,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CACV;AAEvC,YAAA,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK;AAC1B,YAAA,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE;gBACpB,gBAAgB,GAAG,SAAS;gBAC5B,MAAM,CAAC,SAAS,CAAC;gBACjB;YACF;AAEA,YAAA,IAAI,KAAK,CAAC,EAAE,KAAK,eAAe,EAAE;AAChC,gBAAA,eAAe,GAAG,KAAK,CAAC,EAAE;gBAC1B,gBAAgB,GAAG,SAAS;gBAC5B,MAAM,CAAC,SAAS,CAAC;YACnB;AAEA,YAAA,MAAM,eAAe,GAAG;gBACtB,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,WAAW,EAAE,KAAK,CAAC,WAAW;AAC9B,gBAAA,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE;aAC9B;YAED,IAAI,gBAAgB,EAAE;AACpB,gBAAA,MAAM,UAAU,GACd,CAAC,eAAe,CAAC,UAAU,GAAG,gBAAgB,CAAC,UAAU,IAAI,IAAI;gBAEnE,MAAM,SAAS,GACb,eAAe,CAAC,eAAe,GAAG,gBAAgB,CAAC,eAAe;gBAEpE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC;gBAClD,MAAM,CAAC,OAAO,CAAC;YACjB;YACA,gBAAgB,GAAG,eAAe;QACpC,CAAC,EAAE,IAAI,CAAC;AAER,QAAA,OAAO,MAAK;YACV,aAAa,CAAC,UAAU,CAAC;YACzB,MAAM,CAAC,SAAS,CAAC;AACnB,QAAA,CAAC;IACH,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;AAEpC,IAAA,OAAO,GAAG;AACZ,CAAC;AAED;;;;;AAKG;AACI,MAAM,yBAAyB,GAAG,CACvC,KAKC,KACC;IACF,MAAM,EACJ,eAAe,EACf,QAAQ,EACR,gBAAgB,GAAG,uBAAuB,EAC1C,gBAAgB,EAAE,iBAAiB,GAAG,SAAS,EAC/C,eAAe,EAAE,gBAAgB,GAAG,SAAS,EAC7C,mBAAmB,EAAE,oBAAoB,GAAG,SAAS,EACrD,UAAU,EACV,aAAa,EACb,eAAe,EACf,QAAQ,EACR,OAAO,EACP,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,GACpB,GAAG,KAAK;AAET,IAAA,MAAM,IAAI,GAAGC,0BAAO,EAAE;AACtB,IAAA,MAAM,EAAE,kBAAkB,EAAE,GAAGH,oCAAiB,EAAE;AAClD,IAAA,MAAM,eAAe,GAAG,kBAAkB,EAAE;IAE5C,MAAM,CAAC,gBAAgB,EAAE,mBAAmB,CAAC,GAAGC,cAAQ,CAAC,iBAAiB,CAAC;IAC3E,MAAM,CAAC,eAAe,EAAE,kBAAkB,CAAC,GAAGA,cAAQ,CAAC,gBAAgB,CAAC;IACxE,MAAM,CAAC,mBAAmB,EAAE,sBAAsB,CAAC,GACjDA,cAAQ,CAAC,oBAAoB,CAAC;IAEhC,MAAM,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,GAAGA,cAAQ,CAAU,KAAK,CAAC;IAC1E,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAGA,cAAQ,CAAU,KAAK,CAAC;AAE1D,IAAA,MAAM,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC;IAC7C,MAAM,eAAe,GAAG,eAAe,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS;AAE1E,IAAA,MAAM,SAAS,GAAG,OAAO,IAAI,eAAe;AAC5C,IAAA,MAAM,cAAc,GAAGG,YAAM,CAAqB,SAAS,CAAC;AAE5D,IAAA,MAAM,WAAW,GAAGC,iBAAW,CAC7B,CAAC,KAAuB,KAAI;AAC1B,QAAA,MAAM,GAAG,GAAG,KAAK,EAAE,GAAG;AACtB,QAAA,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE;AAC7B,YAAA,cAAc,CAAC,OAAO,GAAG,SAAS;YAClC,oBAAoB,CAAC,KAAK,CAAC;YAC3B;QACF;AAEA,QAAA,MAAM,KAAK,GAAG,GAAG,GAAG,SAAS;AAC7B,QAAA,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,IAAI,KAAK;AAEpD,QAAA,MAAM,eAAe,GAAG,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,IAAI,YAAY;AAClE,QAAA,cAAc,CAAC,OAAO,GAAG,eAAe;AAExC,QAAA,oBAAoB,CAAC,CAAC,IAAI,KAAI;AAC5B,YAAA,IAAI,IAAI,IAAI,eAAe,GAAG,iCAAiC;AAC7D,gBAAA,OAAO,KAAK;AACd,YAAA,IAAI,CAAC,IAAI,IAAI,eAAe,GAAG,iCAAiC;AAC9D,gBAAA,OAAO,IAAI;AAEb,YAAA,OAAO,IAAI;AACb,QAAA,CAAC,CAAC;AACJ,IAAA,CAAC,EACD,CAAC,SAAS,CAAC,CACZ;AAED,IAAA,MAAM,WAAW,GAAiCC,aAAO,CAAC,MAAK;QAC7D,IAAI,CAAC,gBAAgB,EAAE;AACrB,YAAA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE;QAC5B;QAEA,MAAM,OAAO,GAAwC,EAAE;QAEvD,IAAI,iBAAiB,EAAE;AACrB,YAAA,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC;QAC5B;AAEA,QAAA,MAAM,wBAAwB,GAC5B,eAAe,EAAE,cAAc,EAAE,wBAAwB;AAE3D,QAAA,IACE,iBAAiB;YACjB,wBAAwB;AACxB,YAAA,wBAAwB,EAAE,QAAQ,CAAC,KAAK,CAAC,EACzC;AACA,YAAA,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAChC;QAEA,OAAO;AACL,YAAA,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;AAC5B,YAAA,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,GAAG,OAAO,GAAG,SAAS;SACjD;AACH,IAAA,CAAC,EAAE;QACD,iBAAiB;QACjB,eAAe,EAAE,cAAc,EAAE,wBAAwB;QACzD,gBAAgB;AACjB,KAAA,CAAC;AAEF,IAAA,MAAM,eAAe,GAAGF,YAAM,CAAsB,SAAS,CAAC;IAC9DF,eAAS,CAAC,MAAK;AACb,QAAA,MAAM,eAAe,GAAG,WAAW,CAAC,QAAQ;AAC5C,QAAA,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO;QAE5C,IACE,CAAC,CAAC,gBAAgB;AAClB,YAAA,YAAY,KAAK,SAAS;YAC1B,YAAY,KAAK,eAAe,EAChC;AACA,YAAA,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE;AAClD,gBAAA,QAAQ,EAAE,eAAe;gBACzB,MAAM,EAAE,WAAW,EAAE,MAAM;gBAC3B,KAAK,EAAE,cAAc,CAAC,OAAO;gBAC7B,SAAS;AACV,aAAA,CAAC;QACJ;AACA,QAAA,eAAe,CAAC,OAAO,GAAG,eAAe;AAC3C,IAAA,CAAC,EAAE;AACD,QAAA,WAAW,CAAC,QAAQ;AACpB,QAAA,WAAW,CAAC,MAAM;QAClB,gBAAgB;AAChB,QAAA,IAAI,EAAE,MAAM;QACZ,SAAS;AACV,KAAA,CAAC;AAEF,IAAA,MAAM,0BAA0B,GAAGG,iBAAW,CAAC,CAAC,QAAgB,KAAI;QAClE,mBAAmB,CAAC,OAAO,CAAC;QAC5B,kBAAkB,CAAC,QAAQ,CAAC;IAC9B,CAAC,EAAE,EAAE,CAAC;IAEN,MAAM,yBAAyB,GAAGA,iBAAW,CAC3C,CAAC,SAAA,GAAiC,MAAM,KAAI;QAC1C,mBAAmB,CAAC,MAAM,CAAC;QAC3B,sBAAsB,CAAC,SAAS,CAAC;IACnC,CAAC,EACD,EAAE,CACH;AAED,IAAA,MAAM,uBAAuB,GAAGA,iBAAW,CAAC,MAAK;QAC/C,mBAAmB,CAAC,SAAS,CAAC;QAC9B,kBAAkB,CAAC,SAAS,CAAC;QAC7B,sBAAsB,CAAC,SAAS,CAAC;AAEjC,QAAA,cAAc,CAAC,OAAO,GAAG,SAAS;QAClC,oBAAoB,CAAC,KAAK,CAAC;IAC7B,CAAC,EAAE,EAAE,CAAC;AAEN,IAAA,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAGJ,cAAQ,CAAe,YAAY,CAAC,IAAI,CAAC;IACrE,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAGA,cAAQ,CAAC,KAAK,CAAC;IACrDC,eAAS,CAAC,MAAK;AACb,QAAA,eAAe,CACb,eAAe,EACf,kBAAkB,EAClB,kBAAkB,CACnB,CAAC,IAAI,CAAC,CAAC,gBAAgB,KAAI;YAC1B,SAAS,CAAC,gBAAgB,CAAC;AAC3B,YAAA,cAAc,CAAC,gBAAgB,KAAK,YAAY,CAAC,IAAI,CAAC;AACxD,QAAA,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,CAAC,CAAC;IAE7D,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAGD,cAAQ,EAAU;IAC9CC,eAAS,CAAC,MAAK;AACb,QAAA,IAAI,MAAM,KAAK,YAAY,CAAC,EAAE;YAAE;QAEhCK,0BAAU,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE;aAC/C,IAAI,CAAC,SAAS;AACd,aAAA,KAAK,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IAEjD,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAGN,cAAQ,EAAe;IACzDC,eAAS,CAAC,MAAK;AACb,QAAA,IAAI,MAAM,KAAK,YAAY,CAAC,UAAU;YAAE;AAExC,QAAAM,6BAAa,CAAC;AACZ,YAAA,QAAQ,EAAE,QAAQ;AAClB,YAAA,SAAS,EAAE,aAAa;SACzB;aACE,IAAI,CAAC,YAAY;AACjB,aAAA,KAAK,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC;IACtE,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAErC,IAAA,MAAM,WAAW,GAAGH,iBAAW,CAC7B,CAAC,KAAU,KAAI;AACb,QAAA,OAAO,CAAC,IAAI,CACV,4DAA4D,CAC7D;AACD,QAAA,uBAAuB,EAAE;AACzB,QAAA,OAAO,GAAG,KAAK,CAAC;AAClB,IAAA,CAAC,EACD,CAAC,uBAAuB,EAAE,OAAO,CAAC,CACnC;AAED,IAAA,MAAM,OAAO,GAAG,eAAe,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,SAAS;AAExD,IAAA,MAAM,YAAY,GAAGC,aAAO,CAC1B,OAAO;QACL,WAAW;QACX,WAAW;QACX,OAAO;QACP,SAAS;QACT,eAAe;QACf,mBAAmB;QACnB,gBAAgB;QAChB,uBAAuB;QACvB,yBAAyB;QACzB,0BAA0B;QAC1B,gBAAgB;QAChB,UAAU;QACV,aAAa;QACb,QAAQ;AACR,QAAA,OAAO,EAAE,WAAW;QACpB,mBAAmB;AACpB,KAAA,CAAC,EACF;QACE,WAAW;QACX,WAAW;QACX,OAAO;QACP,SAAS;QACT,eAAe;QACf,mBAAmB;QACnB,gBAAgB;QAChB,uBAAuB;QACvB,yBAAyB;QACzB,0BAA0B;QAC1B,gBAAgB;QAChB,UAAU;QACV,aAAa;QACb,QAAQ;QACR,WAAW;QACX,mBAAmB;AACpB,KAAA,CACF;AACD,IAAA,QACEG,eAAA,CAAC,eAAe,CAAC,QAAQ,EAAA,EAAC,KAAK,EAAE,YAAY,EAAA,QAAA,EAAA,CAC1C,QAAQ,EACR,OAAO,KACNC,cAAA,CAAC,iBAAiB,IAChB,GAAG,EAAE,YAAY,EACjB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,WAAW,EACpB,YAAY,EAAE,YAAY,GAC1B,CACH,CAAA,EAAA,CACwB;AAE/B;AAEA,MAAM,iBAAiB,GAAG,CAAC,KAM1B,KAAI;AACH,IAAA,MAAM,IAAI,GAAGP,0BAAO,EAAE;AACtB,IAAA,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,KAAK;AAC5D,IAAA,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC;AAClE,IAAA,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,GAAG;AACzC,IAAA,MAAM,cAAc,GAAGC,YAAM,CAAqC,SAAS,CAAC;AAC5E,IAAA,cAAc,CAAC,OAAO,GAAG,OAAO;AAEhC,IAAA,MAAM,cAAc,GAAGA,YAAM,CAE3B,SAAS,CAAC;AACZ,IAAA,cAAc,CAAC,OAAO,GAAG,OAAO;AAEhC,IAAA,MAAM,YAAY,GAAG,CAAC,CAAC,gBAAgB;IAEvCF,eAAS,CAAC,MAAK;AACb,QAAA,IAAI,CAAC,IAAI,IAAI,CAAC,YAAY;YAAE;QAE5B,YAAY,CAAC,IAAI,CAAC;AAClB,QAAA,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,EAAE,KAAI;AACnE,YAAA,OAAO,KAAK,CACV,EAAE,EACF,CAAC,KAAK,KAAK,cAAc,CAAC,OAAO,GAAG,KAAK,CAAC,EAC1C,CAAC,KAAuB,KAAK,cAAc,CAAC,OAAO,GAAG,KAAK,CAAC,CAC7D;AACH,QAAA,CAAC,CAAC;AACF,QAAA,UAAU,CAAC,OAAO,CAAC,MAAK;YACtB,YAAY,CAAC,KAAK,CAAC;AACrB,QAAA,CAAC,CAAC;AAEF,QAAA,OAAO,MAAK;AACV,YAAA,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA,uBAAA,CAAyB,EAAE,GAAG,CAAC,CAAC;AAC3E,QAAA,CAAC;IACH,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;AAE7C,IAAA,OAAO,QAAQ;AACjB,CAAC;AAED,MAAM,WAAW,GAAG,CAClB,GAAkC,EAClC,MAA0B,EAC1B,IAAsB,EACtB,MAAoB,KAClB;AACF,IAAA,MAAM,EACJ,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,aAAa,EACb,QAAQ,EACR,mBAAmB,GACpB,GAAG,GAAG;AAEP,IAAA,MAAM,UAAU,GAAGE,YAAM,CAA0B,EAAE,CAAC;IACtD,UAAU,CAAC,OAAO,GAAG;QACnB,gBAAgB;QAChB,mBAAmB;QACnB,eAAe;QACf,mBAAmB;KACpB;AAED,IAAA,MAAM,YAAY,GAAGA,YAAM,CAAgC,SAAS,CAAC;AAErE,IAAA,MAAM,QAAQ,GAAGA,YAAM,CAAmB,IAAI,CAAC;AAC/C,IAAA,MAAM,SAAS,GAAGA,YAAM,CAAoB,IAAI,CAAC;AACjD,IAAA,MAAM,UAAU,GAAGA,YAAM,CAAmB,IAAI,CAAC;AACjD,IAAA,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAGH,cAAQ,CACxC;AACE,QAAA,KAAK,EAAE,IAAI;AACX,QAAA,MAAM,EAAE,IAAI;AACb,KAAA,CACF;IAED,MAAM,cAAc,GAAGI,iBAAW,CAChC,CACE,EAAe,EACf,OAA8B,EAC9B,OAA2C,KACzC;AACF,QAAA,IAAI,YAAqC;AACzC,QAAA,IAAI,SAAwC;QAE5C,MAAM,MAAM,GAAG,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,MAAM,KAAI;AAC1D,YAAA,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO;AAClC,YAAA,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE;AAC7B,gBAAA,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBACxC;YACF;YAEA,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,cAAc,EAAE;YACnC,IAAI,CAAC,KAAK,EAAE;AACV,gBAAA,MAAM,CAAC,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBAC1D;YACF;AAEA,YAAA,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;AAC7C,gBAAA,GAAG,OAAO;gBACV,MAAM,EAAE,YAAY,CAAC,UAAU;gBAC/B,aAAa;AACd,aAAA,CAAC;AAEF,YAAA,MAAM,aAAa,GAAG,KAAK,CAAC,WAAW,EAAE;AACzC,YAAAM,kBAAS,CAAC,MACR,YAAY,CAAC;AACX,gBAAA,KAAK,EAAE,aAAa,CAAC,KAAK,IAAI,CAAC;AAC/B,gBAAA,MAAM,EAAE,aAAa,CAAC,MAAM,IAAI,CAAC;AAClC,aAAA,CAAC,CACH;YAED,SAAS,GAAG,IAAIC,iCAAiB,CAC/B,KAAK,EACL,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,EAClD,EAAE,OAAO,EAAE,OAAO,EAAE,CACrB;AACD,YAAA,YAAY,CAAC,OAAO,GAAG,SAAS;YAChC;AACG,iBAAA,KAAK;AACL,iBAAA,IAAI,CAAC,CAAC,cAAc,KAAI;gBACvB,YAAY,GAAG,IAAI,WAAW,CAAC,CAAC,cAAc,CAAC,CAAC;gBAChD,OAAO,CAAC,YAAY,CAAC;AACvB,YAAA,CAAC;iBACA,KAAK,CAAC,MAAM,CAAC;AAClB,QAAA,CAAC,CAAC;QAEF,OAAO;YACL,MAAM;YACN,IAAI,EAAE,MAAK;gBACT,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,IAAI,CAAC;AACrD,gBAAA,YAAY,CAAC,OAAO,GAAG,SAAS;gBAChC,SAAS,EAAE,IAAI,EAAE;AACjB,gBAAA,IAAI,YAAY;oBAAEC,gCAAoB,CAAC,YAAY,CAAC;YACtD,CAAC;SACF;IACH,CAAC,EACD,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,CACxC;IAED,MAAM,OAAO,GAAGR,iBAAW,CACzB,CAAC,EAAe,EAAE,OAA8B,KAAI;AAClD,QAAA,IAAI,YAAqC;AACzC,QAAA,IAAI,QAA8B;QAElC,MAAM,MAAM,GAAG,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,MAAM,KAAI;YAC1D,IAAI,CAAC,gBAAgB,EAAE;AACrB,gBAAA,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBACxC;YACF;AAEA,YAAA,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO;AAChC,YAAA,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO;AAClC,YAAA,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO;YAEpC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,cAAc,EAAE;YACnC,IAAI,CAAC,KAAK,EAAE;AACV,gBAAA,MAAM,CAAC,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBAC1D;YACF;AAEA,YAAA,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,EAAE;AAC5D,gBAAA,MAAM,CAAC,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;gBAC/D;YACF;AAEA,YAAA,OAAO,CAAC,SAAS,GAAG,EAAE;AACtB,YAAA,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CACjB,MAAK;AACH,gBAAA,MAAM,aAAa,GAAG,KAAK,CAAC,WAAW,EAAE;AACzC,gBAAAM,kBAAS,CAAC,MACR,YAAY,CAAC;AACX,oBAAA,KAAK,EAAE,aAAa,CAAC,KAAK,IAAI,CAAC;AAC/B,oBAAA,MAAM,EAAE,aAAa,CAAC,MAAM,IAAI,CAAC;AAClC,iBAAA,CAAC,CACH;AACD,gBAAA,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;oBAC7C,gBAAgB;oBAChB,mBAAmB;oBACnB,eAAe;oBACf,MAAM,EAAE,YAAY,CAAC,EAAE;AACxB,iBAAA,CAAC;gBAEF,IAAI,CAAC,MAAM,EAAE;AACX,oBAAA,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;oBAC/C;gBACF;gBAEA,QAAQ,GAAGG,8BAAc,CACvB,MAAM,EACN,OAAO,EACP,QAAQ,EACR;oBACE,gBAAgB;oBAChB,mBAAmB;oBACnB,eAAe,EAAE,SAAS,IAAI,SAAS;iBACxC,EACD,OAAO,CACR;AACD,gBAAA,YAAY,GAAG,QAAQ,CAAC,aAAa,EAAE;gBAEvC,OAAO,CAAC,YAAY,CAAC;YACvB,CAAC,EACD,MAAK;AACH,gBAAA,MAAM,CAAC,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;AAC7D,YAAA,CAAC,CACF;AACH,QAAA,CAAC,CAAC;QAEF,OAAO;YACL,MAAM;YACN,IAAI,EAAE,MAAK;gBACT,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,IAAI,CAAC;gBACrD,QAAQ,EAAE,OAAO,EAAE;gBACnB,IAAI,QAAQ,CAAC,OAAO;AAAE,oBAAA,QAAQ,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI;AACvD,gBAAA,IAAI,YAAY;oBAAED,gCAAoB,CAAC,YAAY,CAAC;YACtD,CAAC;SACF;AACH,IAAA,CAAC,EACD;AACE,QAAA,IAAI,EAAE,MAAM;QACZ,MAAM;QACN,gBAAgB;QAChB,mBAAmB;QACnB,eAAe;AAChB,KAAA,CACF;AAED,IAAA,MAAM,KAAK,GAAG,MAAM,KAAK,YAAY,CAAC,EAAE,GAAG,OAAO,GAAG,cAAc;IAEnEX,eAAS,CAAC,MAAK;AACb,QAAA,IAAI,CAAC,gBAAgB;YAAE;AAEvB,QAAA,YAAY,CAAC;AACX,cAAE,aAAa,CAAC;YACd,gBAAgB;YAChB,mBAAmB;YACnB,eAAe;YACf,mBAAmB;SACpB;AACA,aAAA,KAAK,CAAC,CAAC,GAAG,KACT,OAAO,CAAC,IAAI,CAAC,CAAA,4CAAA,CAA8C,EAAE,GAAG,CAAC,CAClE;AACL,IAAA,CAAC,EAAE;QACD,gBAAgB;QAChB,mBAAmB;QACnB,eAAe;QACf,mBAAmB;AACpB,KAAA,CAAC;IAEF,MAAM,QAAQ,IACZO,eAAA,CAAA,KAAA,EAAA,EAAK,SAAS,EAAC,+BAA+B,aAC5CC,cAAA,CAAA,OAAA,EAAA,EACE,SAAS,EAAE,IAAI,CACb,sCAAsC,EACtC,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,KAAK;oBAChC,4CAA4C,CAC/C,EACD,GAAG,EAAE,QAAQ,EACb,WAAW,EAAA,IAAA,EACX,KAAK,EAAA,IAAA,EACL,QAAQ,EAAE,KAAK,EAAA,GACX,SAAS,EAAA,CACb,EACD,eAAe,KACdA,cAAA,CAAA,KAAA,EAAA,EACE,SAAS,EAAC,iDAAiD,EAC3D,GAAG,EAAC,YAAY,EAChB,GAAG,EAAE,UAAU,EACf,WAAW,EAAC,WAAW,EACvB,GAAG,EAAE,eAAe,EAAA,GAChB,SAAS,EAAA,CACb,CACH,EACDA,cAAA,CAAA,QAAA,EAAA,EACE,SAAS,EAAC,8CAA8C,EAAA,GACpD,SAAS,EACb,GAAG,EAAE,SAAS,EAAA,CACd,CAAA,EAAA,CACE,CACP;AAED,IAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE;AAC5B,CAAC;;;;"}
|
|
@@ -276,8 +276,9 @@ const BackgroundFilters = (props) => {
|
|
|
276
276
|
handleErrorRef.current = onError;
|
|
277
277
|
const handleStatsRef = react.useRef(undefined);
|
|
278
278
|
handleStatsRef.current = onStats;
|
|
279
|
+
const filterActive = !!backgroundFilter;
|
|
279
280
|
react.useEffect(() => {
|
|
280
|
-
if (!call || !
|
|
281
|
+
if (!call || !filterActive)
|
|
281
282
|
return;
|
|
282
283
|
setIsLoading(true);
|
|
283
284
|
const { unregister, registered } = call.camera.registerFilter((ms) => {
|
|
@@ -289,11 +290,19 @@ const BackgroundFilters = (props) => {
|
|
|
289
290
|
return () => {
|
|
290
291
|
unregister().catch((err) => console.warn(`Can't unregister filter`, err));
|
|
291
292
|
};
|
|
292
|
-
}, [call, start,
|
|
293
|
+
}, [call, start, filterActive, setIsLoading]);
|
|
293
294
|
return children;
|
|
294
295
|
};
|
|
295
296
|
const useRenderer = (api, tfLite, call, engine) => {
|
|
296
297
|
const { backgroundFilter, backgroundBlurLevel, backgroundImage, modelFilePath, basePath, segmentationOptions, } = api;
|
|
298
|
+
const optionsRef = react.useRef({});
|
|
299
|
+
optionsRef.current = {
|
|
300
|
+
backgroundFilter,
|
|
301
|
+
backgroundBlurLevel,
|
|
302
|
+
backgroundImage,
|
|
303
|
+
segmentationOptions,
|
|
304
|
+
};
|
|
305
|
+
const processorRef = react.useRef(undefined);
|
|
297
306
|
const videoRef = react.useRef(null);
|
|
298
307
|
const canvasRef = react.useRef(null);
|
|
299
308
|
const bgImageRef = react.useRef(null);
|
|
@@ -301,9 +310,53 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
301
310
|
width: 1920,
|
|
302
311
|
height: 1080,
|
|
303
312
|
});
|
|
304
|
-
const
|
|
313
|
+
const startMediaPipe = react.useCallback((ms, onError, onStats) => {
|
|
305
314
|
let outputStream;
|
|
306
315
|
let processor;
|
|
316
|
+
const output = new Promise((resolve, reject) => {
|
|
317
|
+
const options = optionsRef.current;
|
|
318
|
+
if (!options.backgroundFilter) {
|
|
319
|
+
reject(new Error('No filter specified'));
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
const [track] = ms.getVideoTracks();
|
|
323
|
+
if (!track) {
|
|
324
|
+
reject(new Error('No video tracks in input media stream'));
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
call?.tracer.trace('backgroundFilters.enable', {
|
|
328
|
+
...options,
|
|
329
|
+
engine: FilterEngine.MEDIA_PIPE,
|
|
330
|
+
modelFilePath,
|
|
331
|
+
});
|
|
332
|
+
const trackSettings = track.getSettings();
|
|
333
|
+
reactDom.flushSync(() => setVideoSize({
|
|
334
|
+
width: trackSettings.width ?? 0,
|
|
335
|
+
height: trackSettings.height ?? 0,
|
|
336
|
+
}));
|
|
337
|
+
processor = new videoFiltersWeb.VirtualBackground(track, { ...options, basePath, modelPath: modelFilePath }, { onError, onStats });
|
|
338
|
+
processorRef.current = processor;
|
|
339
|
+
processor
|
|
340
|
+
.start()
|
|
341
|
+
.then((processedTrack) => {
|
|
342
|
+
outputStream = new MediaStream([processedTrack]);
|
|
343
|
+
resolve(outputStream);
|
|
344
|
+
})
|
|
345
|
+
.catch(reject);
|
|
346
|
+
});
|
|
347
|
+
return {
|
|
348
|
+
output,
|
|
349
|
+
stop: () => {
|
|
350
|
+
call?.tracer.trace('backgroundFilters.disable', null);
|
|
351
|
+
processorRef.current = undefined;
|
|
352
|
+
processor?.stop();
|
|
353
|
+
if (outputStream)
|
|
354
|
+
videoClient.disposeOfMediaStream(outputStream);
|
|
355
|
+
},
|
|
356
|
+
};
|
|
357
|
+
}, [call?.tracer, modelFilePath, basePath]);
|
|
358
|
+
const startTf = react.useCallback((ms, onError) => {
|
|
359
|
+
let outputStream;
|
|
307
360
|
let renderer;
|
|
308
361
|
const output = new Promise((resolve, reject) => {
|
|
309
362
|
if (!backgroundFilter) {
|
|
@@ -318,83 +371,42 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
318
371
|
reject(new Error('No video tracks in input media stream'));
|
|
319
372
|
return;
|
|
320
373
|
}
|
|
321
|
-
if (
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
modelFilePath,
|
|
328
|
-
});
|
|
329
|
-
if (!videoEl) {
|
|
330
|
-
reject(new Error('Renderer started before elements are ready'));
|
|
331
|
-
return;
|
|
332
|
-
}
|
|
374
|
+
if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
|
|
375
|
+
reject(new Error('Renderer started before elements are ready'));
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
videoEl.srcObject = ms;
|
|
379
|
+
videoEl.play().then(() => {
|
|
333
380
|
const trackSettings = track.getSettings();
|
|
334
381
|
reactDom.flushSync(() => setVideoSize({
|
|
335
382
|
width: trackSettings.width ?? 0,
|
|
336
383
|
height: trackSettings.height ?? 0,
|
|
337
384
|
}));
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
modelPath: modelFilePath,
|
|
385
|
+
call?.tracer.trace('backgroundFilters.enable', {
|
|
386
|
+
backgroundFilter,
|
|
341
387
|
backgroundBlurLevel,
|
|
342
388
|
backgroundImage,
|
|
343
|
-
|
|
344
|
-
segmentationOptions,
|
|
345
|
-
}, { onError, onStats });
|
|
346
|
-
processor
|
|
347
|
-
.start()
|
|
348
|
-
.then((processedTrack) => {
|
|
349
|
-
outputStream = new MediaStream([processedTrack]);
|
|
350
|
-
resolve(outputStream);
|
|
351
|
-
})
|
|
352
|
-
.catch((error) => {
|
|
353
|
-
reject(error);
|
|
389
|
+
engine: FilterEngine.TF,
|
|
354
390
|
});
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
if (engine === FilterEngine.TF) {
|
|
358
|
-
if (!videoEl || !canvasEl || (backgroundImage && !bgImageEl)) {
|
|
359
|
-
reject(new Error('Renderer started before elements are ready'));
|
|
391
|
+
if (!tfLite) {
|
|
392
|
+
reject(new Error('TensorFlow Lite not loaded'));
|
|
360
393
|
return;
|
|
361
394
|
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
backgroundImage,
|
|
373
|
-
engine,
|
|
374
|
-
});
|
|
375
|
-
if (!tfLite) {
|
|
376
|
-
reject(new Error('TensorFlow Lite not loaded'));
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
renderer = videoFiltersWeb.createRenderer(tfLite, videoEl, canvasEl, {
|
|
380
|
-
backgroundFilter,
|
|
381
|
-
backgroundBlurLevel,
|
|
382
|
-
backgroundImage: bgImageEl ?? undefined,
|
|
383
|
-
}, onError);
|
|
384
|
-
outputStream = canvasEl.captureStream();
|
|
385
|
-
resolve(outputStream);
|
|
386
|
-
}, () => {
|
|
387
|
-
reject(new Error('Could not play the source video stream'));
|
|
388
|
-
});
|
|
389
|
-
return;
|
|
390
|
-
}
|
|
391
|
-
reject(new Error('No supported engine available'));
|
|
395
|
+
renderer = videoFiltersWeb.createRenderer(tfLite, videoEl, canvasEl, {
|
|
396
|
+
backgroundFilter,
|
|
397
|
+
backgroundBlurLevel,
|
|
398
|
+
backgroundImage: bgImageEl ?? undefined,
|
|
399
|
+
}, onError);
|
|
400
|
+
outputStream = canvasEl.captureStream();
|
|
401
|
+
resolve(outputStream);
|
|
402
|
+
}, () => {
|
|
403
|
+
reject(new Error('Could not play the source video stream'));
|
|
404
|
+
});
|
|
392
405
|
});
|
|
393
406
|
return {
|
|
394
407
|
output,
|
|
395
408
|
stop: () => {
|
|
396
409
|
call?.tracer.trace('backgroundFilters.disable', null);
|
|
397
|
-
processor?.stop();
|
|
398
410
|
renderer?.dispose();
|
|
399
411
|
if (videoRef.current)
|
|
400
412
|
videoRef.current.srcObject = null;
|
|
@@ -403,14 +415,28 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
403
415
|
},
|
|
404
416
|
};
|
|
405
417
|
}, [
|
|
418
|
+
call?.tracer,
|
|
419
|
+
tfLite,
|
|
420
|
+
backgroundFilter,
|
|
406
421
|
backgroundBlurLevel,
|
|
422
|
+
backgroundImage,
|
|
423
|
+
]);
|
|
424
|
+
const start = engine === FilterEngine.TF ? startTf : startMediaPipe;
|
|
425
|
+
react.useEffect(() => {
|
|
426
|
+
if (!backgroundFilter)
|
|
427
|
+
return;
|
|
428
|
+
processorRef.current
|
|
429
|
+
?.updateOptions({
|
|
430
|
+
backgroundFilter,
|
|
431
|
+
backgroundBlurLevel,
|
|
432
|
+
backgroundImage,
|
|
433
|
+
segmentationOptions,
|
|
434
|
+
})
|
|
435
|
+
.catch((err) => console.warn(`[filters] Can't apply the background options`, err));
|
|
436
|
+
}, [
|
|
407
437
|
backgroundFilter,
|
|
438
|
+
backgroundBlurLevel,
|
|
408
439
|
backgroundImage,
|
|
409
|
-
call?.tracer,
|
|
410
|
-
tfLite,
|
|
411
|
-
engine,
|
|
412
|
-
modelFilePath,
|
|
413
|
-
basePath,
|
|
414
440
|
segmentationOptions,
|
|
415
441
|
]);
|
|
416
442
|
const children = (jsxRuntime.jsxs("div", { className: "str-video__background-filters", children: [jsxRuntime.jsx("video", { className: clsx('str-video__background-filters__video', videoSize.height > videoSize.width &&
|
|
@@ -419,4 +445,4 @@ const useRenderer = (api, tfLite, call, engine) => {
|
|
|
419
445
|
};
|
|
420
446
|
|
|
421
447
|
exports.BackgroundFiltersProvider = BackgroundFiltersProvider;
|
|
422
|
-
//# sourceMappingURL=embedded-BackgroundFilters-
|
|
448
|
+
//# sourceMappingURL=embedded-BackgroundFilters-DMStbSEg.cjs.js.map
|