@100mslive/hms-virtual-background 1.13.25-alpha.0 → 1.13.25-alpha.10

package/README.md

@@ -1,13 +1,243 @@
-## installation
+# Virtual Background with Effects SDK
 
-**with npm:**
+[![Lint, Test and Build](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml/badge.svg)](https://github.com/100mslive/web-sdks/actions/workflows/lint-test-build.yml)
+[![Bundle Size](https://badgen.net/bundlephobia/minzip/@100mslive/hms-virtual-background)](https://bundlephobia.com/result?p=@100mslive/hms-virtual-background)
+[![License](https://img.shields.io/npm/l/@100mslive/hms-virtual-background)](https://www.100ms.live/)
+![Tree shaking](https://badgen.net/bundlephobia/tree-shaking/@100mslive/hms-virtual-background)
 
-```npm i --save @100mslive/hms-virtual-background```
+Virtual background plugin helps in customising one’s background. The customising options are blurring the background or replacing it with a static image. This guide provides an overview of usage of the virtual background plugin of 100ms.
 
-**with yarn**
+## Pre-requisites
 
-```yarn add @100mslive/hms-virtual-background```
+Get the 100ms VirtualBackground Package** (Supported since version 1.11.28)
 
-## Usage
+```bash section=GetHMSVirtualBackgroundPackage sectionIndex=1
+npm install --save @100mslive/hms-virtual-background@latest
+```
 
-Refer our [docs](https://www.100ms.live/docs/javascript/v2/plugins/virtual-background#start-and-stop-virtual-background) for usage.
+## Features
+
+The following features are currently supported under the `HMSEffectsPlugin` class:
+
+```js
+/**
+ * Sets the blur intensity.
+ * @param {number} blur - The blur intensity, ranging from 0 to 1.
+ * @returns {void}
+ */
+setBlur(blur) {}
+
+/**
+ * Sets the virtual background using the provided media URL.
+ * @param {HMSEffectsBackground} url - The media URL to set as the virtual background.
+ * Alternatively, the background image can be downloaded beforehand and passed to setBackground as objectURL
+ * @returns {void}
+ */
+setBackground(url) {}
+
+/**
+ * Retrieves the name of the plugin.
+ * @returns {string} The name of the plugin, 'HMSEffects'.
+ */
+getName() {}
+
+/**
+ * Retrieves the currently enabled background type or media URL.
+ * @returns {string|MediaStream|MediaStreamTrack|HTMLVideoElement} The background type or media URL.
+ */
+getBackground() {}
+
+/**
+ * Sets the preset quality of the virtual background.
+ * Options:  "balanced" | "quality"
+ * The 'quality' preset has a higher CPU usage than the 'balanced' preset which is the default
+ * @param {string} preset - The preset quality to set.
+ * @returns {Promise<void>}
+ */
+setPreset(preset) {}
+
+/**
+ * Retrieves the active preset quality of the virtual background.
+ * @returns {string} The active preset quality.
+ */
+getPreset() {}
+
+/**
+ * Clears all applied filters.
+ * @returns {void}
+ */
+removeEffects() {}
+
+/**
+ * Stops the plugin.
+ * @returns {void}
+ */
+stop() {}
+
+```
+
+Callbacks supported by the plugin:
+
+On initialization, after the required resources are downloaded by the plugin:
+
+```
+const effectsPlugin = new HMSEffectsPlugin(<key>, () => console.log("Plugin initialised"));
+
+```
+
+On resolution change (on device rotation or when the video aspect ratio changes):
+
+```
+effectsPlugin.onResolutionChange = (width: number, height: number) => {
+  console.log(`Resolution changed to ${width}x${height}`)
+}
+```
+
+## Instantiate Virtual Background
+
+The SDK key for effects is needed to instantiate the `HMSEffectsPlugin` class:
+
+```jsx
+  const effectsKey = useHMSStore(selectEffectsKey);
+```
+
+It is recommended to initialise the object in a separate file to prevent multiple initialisations on re-renders and keep the UI level code independent of internal calls by the SDK.
+
+
+```js
+import { HMSEffectsPlugin, HMSVirtualBackgroundTypes } from '@100mslive/hms-virtual-background';
+
+export class VBPlugin {
+  private effectsPlugin?: HMSEffectsPlugin | undefined;
+
+  initialisePlugin = (effectsSDKKey?: string) => {
+    if (this.getVBObject()) {
+      return;
+    }
+    if (effectsSDKKey) {
+      this.effectsPlugin = new HMSEffectsPlugin(effectsSDKKey);
+    }
+  };
+
+  getBackground = () => {
+    return this.effectsPlugin?.getBackground();
+  };
+
+  getBlurAmount = () => {
+    return this.effectsPlugin?.getBlurAmount();
+  };
+
+  getVBObject = () => {
+    return this.effectsPlugin;
+  };
+
+  getName = () => {
+    return this.effectsPlugin?.getName();
+  };
+
+  setBlur = async (blurPower: number) => {
+    this.effectsPlugin?.setBlur(blurPower);
+  };
+
+  setBackground = async (mediaURL: string) => {
+    this.effectsPlugin?.setBackground(mediaURL);
+  };
+
+  setPreset = (preset: string) => {
+    this.effectsPlugin.setPreset(preset);
+  };
+
+  getPreset = () => {
+    return this.effectsPlugin?.getPreset() || '';
+  };
+
+  removeEffects = async () => {
+    this.effectsPlugin?.removeEffects();
+  };
+
+  reset = () => {
+    this.effectsPlugin = undefined;
+  };
+}
+
+export const VBHandler = new VBPlugin();
+```
+
+## Building the UI
+
+The following snippet illustrates how to add the plugin to the video and manage the UI state to preserve configuration:
+
+```jsx
+import {
+  selectEffectsKey,
+  selectIsLocalVideoPluginPresent
+  selectLocalVideoTrackID,
+  useHMSStore,
+} from '@100mslive/react-sdk';
+import { 
+  HMSEffectsPlugin, 
+  HMSVirtualBackgroundTypes
+} from '@100mslive/hms-virtual-background';
+import { VBHandler } from './VBHandler';
+
+export const VirtualBackgroundPicker = () => {
+  const hmsActions = useHMSActions();
+  // Get the effects SDK key here
+  const effectsKey = useHMSStore(selectEffectsKey);
+  const trackId = useHMSStore(selectLocalVideoTrackID);
+  const isPluginAdded = useHMSStore(selectIsLocalVideoPluginPresent(VBHandler?.getName() || ''));
+  
+  // State can be used to show active selection
+  const [background, setBackground] = useState<string | HMSVirtualBackgroundTypes>(
+    VBHandler.getBackground() as string | HMSVirtualBackgroundTypes,
+  );
+
+  useEffect(() => {
+    if (!track?.id) {
+      return;
+    }
+    if (!isPluginAdded) {
+      let vbObject = VBHandler.getVBObject();
+      if (!vbObject) {
+        VBHandler.initialisePlugin(effectsKey);
+        vbObject = VBHandler.getVBObject();
+        if (effectsKey) {
+          hmsActions.addPluginsToVideoStream([vbObject as HMSEffectsPlugin]);
+        } 
+      }
+    }
+  }, [hmsActions, isPluginAdded, effectsKey, track?.id]);
+
+  // UI code for media picker can go here
+}
+
+```
+
+This handles initialisation and adding the plugin to the video stream. The plugin takes a few seconds on first load during initialisation. Subsequent filter and effect selections should take less than a second to reflect.
+
+The methods can be called via the `VBHandler` object:
+```jsx
+const setBackground = async(mediaURL : string) => {
+  await VBHandler?.setBackground(mediaURL);
+  // The selection can be highlighted using the activeBackground state
+  setActiveBackground(mediaURL);
+}
+
+const setBlur = async(blurAmount: number) => {
+  await VBHandler?.setBlur(blurAmount);
+  setActiveBackground(HMSVirtualBackgroundTypes.BLUR);
+}
+
+const removeEffects = async() => {
+  await VBHandler.removeEffects();
+  setActiveBackground(HMSVirtualBackgroundTypes.NONE);
+}
+```
+
+
+The full implementation can be viewed in the [roomkit-react package](https://github.com/100mslive/web-sdks/blob/main/packages/roomkit-react/src/Prebuilt/components/VirtualBackground/VBPicker.tsx).
+
+
+## Supported Browsers
+
+Effects virtual background is supported on Safari, Firefox and Chromium based browsers.<br/>

package/dist/cjs/HMSEffectsPlugin.js

@@ -1,2 +1,2 @@
-"use strict";var h=Object.defineProperty;var g=Object.getOwnPropertyDescriptor;var p=Object.getOwnPropertyNames;var v=Object.prototype.hasOwnProperty;var b=(r,t)=>{for(var e in t)h(r,e,{get:t[e],enumerable:!0})},k=(r,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of p(t))!v.call(r,s)&&s!==e&&h(r,s,{get:()=>t[s],enumerable:!(i=g(t,s))||i.enumerable});return r};var S=r=>k(h({},"__esModule",{value:!0}),r);var f=(r,t,e)=>new Promise((i,s)=>{var d=a=>{try{o(e.next(a))}catch(c){s(c)}},m=a=>{try{o(e.throw(a))}catch(c){s(c)}},o=a=>a.done?i(a.value):Promise.resolve(a.value).then(d,m);o((e=e.apply(r,t)).next())});var E={};b(E,{HMSEffectsPlugin:()=>u});module.exports=S(E);var l=require("effects-sdk");var n="https://assets.100ms.live/effectsdk/3.2.3/";var u=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new l.tsvb(t),this.onInit=e,this.effects.config({sdk_url:n,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${n}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${n}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return f(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};
+"use strict";var h=Object.defineProperty;var g=Object.getOwnPropertyDescriptor;var p=Object.getOwnPropertyNames;var v=Object.prototype.hasOwnProperty;var b=(r,t)=>{for(var e in t)h(r,e,{get:t[e],enumerable:!0})},k=(r,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of p(t))!v.call(r,s)&&s!==e&&h(r,s,{get:()=>t[s],enumerable:!(i=g(t,s))||i.enumerable});return r};var S=r=>k(h({},"__esModule",{value:!0}),r);var f=(r,t,e)=>new Promise((i,s)=>{var d=a=>{try{o(e.next(a))}catch(c){s(c)}},m=a=>{try{o(e.throw(a))}catch(c){s(c)}},o=a=>a.done?i(a.value):Promise.resolve(a.value).then(d,m);o((e=e.apply(r,t)).next())});var E={};b(E,{HMSEffectsPlugin:()=>u});module.exports=S(E);var l=require("effects-sdk");var n="https://assets.100ms.live/effectsdk/3.4.3/";var u=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new l.tsvb(t),this.onInit=e,this.effects.config({sdk_url:n,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${n}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${n}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return f(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};
 //# sourceMappingURL=HMSEffectsPlugin.js.map

package/dist/cjs/HMSEffectsPlugin.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/HMSEffectsPlugin.ts", "../../src/constants.ts"],
-  "sourcesContent": ["import { tsvb } from 'effects-sdk';\nimport { HMSMediaStreamPlugin } from '@100mslive/hms-video-store';\nimport { EFFECTS_SDK_ASSETS } from './constants';\nimport { HMSVirtualBackgroundTypes } from './interfaces';\n\nexport type HMSEffectsBackground = string | MediaStream | MediaStreamTrack | HTMLVideoElement;\n\nexport class HMSEffectsPlugin implements HMSMediaStreamPlugin {\n  private effects: tsvb;\n  // Ranges from 0 to 1, inclusive\n  private blurAmount = 0;\n  private background: HMSEffectsBackground = HMSVirtualBackgroundTypes.NONE;\n  private backgroundType = HMSVirtualBackgroundTypes.NONE;\n  private preset: 'balanced' | 'quality' = 'balanced';\n  private initialised = false;\n  private intervalId: NodeJS.Timer | null = null;\n  private onInit;\n  private onResolutionChangeCallback?: (width: number, height: number) => void;\n  private canvas: HTMLCanvasElement;\n\n  constructor(effectsSDKKey: string, onInit?: () => void) {\n    this.effects = new tsvb(effectsSDKKey);\n    this.onInit = onInit;\n    this.effects.config({\n      sdk_url: EFFECTS_SDK_ASSETS,\n      models: {\n        colorcorrector: '',\n        facedetector: '',\n        lowlighter: '',\n      },\n      wasmPaths: {\n        'ort-wasm.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm.wasm`,\n        'ort-wasm-simd.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd.wasm`,\n      },\n    });\n    this.canvas = document.createElement('canvas');\n    this.effects.onError(err => {\n      // currently logging info type messages as well\n      if (!err.type || err.type === 'error') {\n        console.error('[HMSEffectsPlugin]', err);\n      }\n    });\n    this.effects.cache();\n    this.effects.onReady = () => {\n      if (this.effects) {\n        this.initialised = true;\n        this.onInit?.();\n        this.effects.run();\n        this.effects.setBackgroundFitMode('fill');\n        this.effects.setSegmentationPreset(this.preset);\n        this.applyEffect();\n      }\n    };\n  }\n\n  getName(): string {\n    return 'HMSEffects';\n  }\n\n  private executeAfterInit(callback: () => void) {\n    if (this.initialised) {\n      callback();\n    }\n\n    if (this.intervalId !== null) {\n      clearInterval(this.intervalId);\n    }\n    this.intervalId = setInterval(() => {\n      if (this.initialised) {\n        clearInterval(this.intervalId!);\n        callback();\n      }\n    }, 100);\n  }\n\n  removeBlur() {\n    this.blurAmount = 0;\n    this.executeAfterInit(() => {\n      this.effects.clearBlur();\n    });\n  }\n\n  removeBackground() {\n    this.background = '';\n    this.executeAfterInit(() => {\n      this.effects.clearBackground();\n    });\n  }\n\n  /**\n   * @param blur ranges between 0 and 1\n   */\n  setBlur(blur: number) {\n    this.blurAmount = blur;\n    this.backgroundType = HMSVirtualBackgroundTypes.BLUR;\n    this.removeBackground();\n    this.executeAfterInit(() => {\n      this.effects.setBlur(this.blurAmount);\n    });\n  }\n\n  /**\n   * @param preset can be 'quality' or 'balanced'. The 'quality' preset has better quality but higher CPU usage than 'balanced'\n   */\n  async setPreset(preset: 'quality' | 'balanced') {\n    this.preset = preset;\n    return new Promise((resolve, reject) => {\n      this.executeAfterInit(() => {\n        this.effects.setSegmentationPreset(this.preset).then(resolve).catch(reject);\n      });\n    });\n  }\n\n  onResolutionChange(callback: (width: number, height: number) => void) {\n    this.onResolutionChangeCallback = callback;\n  }\n\n  getPreset() {\n    return this.preset;\n  }\n\n  removeEffects() {\n    this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n    this.removeBackground();\n    this.removeBlur();\n  }\n\n  setBackground(url: HMSEffectsBackground) {\n    this.background = url;\n    this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n    this.removeBlur();\n    this.executeAfterInit(() => {\n      this.effects.setBackground(this.background);\n    });\n  }\n\n  getBlurAmount() {\n    return this.blurAmount;\n  }\n\n  getBackground() {\n    return this.background || this.backgroundType;\n  }\n\n  private updateCanvas(stream: MediaStream) {\n    const { height, width } = stream.getVideoTracks()[0].getSettings();\n    this.canvas.width = width!;\n    this.canvas.height = height!;\n    this.effects.useStream(stream);\n    this.effects.toCanvas(this.canvas);\n  }\n\n  apply(stream: MediaStream): MediaStream {\n    this.effects.clear();\n    this.applyEffect();\n    this.effects.onChangeInputResolution(() => {\n      this.updateCanvas(stream);\n      const { height, width } = stream.getVideoTracks()[0].getSettings();\n      this.onResolutionChangeCallback?.(width!, height!);\n    });\n    this.updateCanvas(stream);\n    return this.canvas.captureStream(30) || stream;\n  }\n\n  stop() {\n    this.removeEffects();\n    this.executeAfterInit(() => {\n      this.effects.stop();\n    });\n  }\n\n  private applyEffect() {\n    if (this.blurAmount) {\n      this.setBlur(this.blurAmount);\n    } else if (this.background) {\n      this.setBackground(this.background);\n    }\n  }\n}\n", "export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/3.2.3/';\n"],
+  "sourcesContent": ["import { tsvb } from 'effects-sdk';\nimport { HMSMediaStreamPlugin } from '@100mslive/hms-video-store';\nimport { EFFECTS_SDK_ASSETS } from './constants';\nimport { HMSVirtualBackgroundTypes } from './interfaces';\n\nexport type HMSEffectsBackground = string | MediaStream | MediaStreamTrack | HTMLVideoElement;\n\nexport class HMSEffectsPlugin implements HMSMediaStreamPlugin {\n  private effects: tsvb;\n  // Ranges from 0 to 1, inclusive\n  private blurAmount = 0;\n  private background: HMSEffectsBackground = HMSVirtualBackgroundTypes.NONE;\n  private backgroundType = HMSVirtualBackgroundTypes.NONE;\n  private preset: 'balanced' | 'quality' = 'balanced';\n  private initialised = false;\n  private intervalId: NodeJS.Timer | null = null;\n  private onInit;\n  private onResolutionChangeCallback?: (width: number, height: number) => void;\n  private canvas: HTMLCanvasElement;\n\n  constructor(effectsSDKKey: string, onInit?: () => void) {\n    this.effects = new tsvb(effectsSDKKey);\n    this.onInit = onInit;\n    this.effects.config({\n      sdk_url: EFFECTS_SDK_ASSETS,\n      models: {\n        colorcorrector: '',\n        facedetector: '',\n        lowlighter: '',\n      },\n      wasmPaths: {\n        'ort-wasm.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm.wasm`,\n        'ort-wasm-simd.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd.wasm`,\n      },\n    });\n    this.canvas = document.createElement('canvas');\n    this.effects.onError(err => {\n      // currently logging info type messages as well\n      if (!err.type || err.type === 'error') {\n        console.error('[HMSEffectsPlugin]', err);\n      }\n    });\n    this.effects.cache();\n    this.effects.onReady = () => {\n      if (this.effects) {\n        this.initialised = true;\n        this.onInit?.();\n        this.effects.run();\n        this.effects.setBackgroundFitMode('fill');\n        this.effects.setSegmentationPreset(this.preset);\n        this.applyEffect();\n      }\n    };\n  }\n\n  getName(): string {\n    return 'HMSEffects';\n  }\n\n  private executeAfterInit(callback: () => void) {\n    if (this.initialised) {\n      callback();\n    }\n\n    if (this.intervalId !== null) {\n      clearInterval(this.intervalId);\n    }\n    this.intervalId = setInterval(() => {\n      if (this.initialised) {\n        clearInterval(this.intervalId!);\n        callback();\n      }\n    }, 100);\n  }\n\n  removeBlur() {\n    this.blurAmount = 0;\n    this.executeAfterInit(() => {\n      this.effects.clearBlur();\n    });\n  }\n\n  removeBackground() {\n    this.background = '';\n    this.executeAfterInit(() => {\n      this.effects.clearBackground();\n    });\n  }\n\n  /**\n   * @param blur ranges between 0 and 1\n   */\n  setBlur(blur: number) {\n    this.blurAmount = blur;\n    this.backgroundType = HMSVirtualBackgroundTypes.BLUR;\n    this.removeBackground();\n    this.executeAfterInit(() => {\n      this.effects.setBlur(this.blurAmount);\n    });\n  }\n\n  /**\n   * @param preset can be 'quality' or 'balanced'. The 'quality' preset has better quality but higher CPU usage than 'balanced'\n   */\n  async setPreset(preset: 'quality' | 'balanced') {\n    this.preset = preset;\n    return new Promise((resolve, reject) => {\n      this.executeAfterInit(() => {\n        this.effects.setSegmentationPreset(this.preset).then(resolve).catch(reject);\n      });\n    });\n  }\n\n  onResolutionChange(callback: (width: number, height: number) => void) {\n    this.onResolutionChangeCallback = callback;\n  }\n\n  getPreset() {\n    return this.preset;\n  }\n\n  removeEffects() {\n    this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n    this.removeBackground();\n    this.removeBlur();\n  }\n\n  setBackground(url: HMSEffectsBackground) {\n    this.background = url;\n    this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n    this.removeBlur();\n    this.executeAfterInit(() => {\n      this.effects.setBackground(this.background);\n    });\n  }\n\n  getBlurAmount() {\n    return this.blurAmount;\n  }\n\n  getBackground() {\n    return this.background || this.backgroundType;\n  }\n\n  private updateCanvas(stream: MediaStream) {\n    const { height, width } = stream.getVideoTracks()[0].getSettings();\n    this.canvas.width = width!;\n    this.canvas.height = height!;\n    this.effects.useStream(stream);\n    this.effects.toCanvas(this.canvas);\n  }\n\n  apply(stream: MediaStream): MediaStream {\n    this.effects.clear();\n    this.applyEffect();\n    this.effects.onChangeInputResolution(() => {\n      this.updateCanvas(stream);\n      const { height, width } = stream.getVideoTracks()[0].getSettings();\n      this.onResolutionChangeCallback?.(width!, height!);\n    });\n    this.updateCanvas(stream);\n    return this.canvas.captureStream(30) || stream;\n  }\n\n  stop() {\n    this.removeEffects();\n    this.executeAfterInit(() => {\n      this.effects.stop();\n    });\n  }\n\n  private applyEffect() {\n    if (this.blurAmount) {\n      this.setBlur(this.blurAmount);\n    } else if (this.background) {\n      this.setBackground(this.background);\n    }\n  }\n}\n", "export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/3.4.3/';\n"],
   "mappings": "snBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,sBAAAE,IAAA,eAAAC,EAAAH,GAAA,IAAAI,EAAqB,uBCAd,IAAMC,EAAqB,6CDO3B,IAAMC,EAAN,KAAuD,CAa5D,YAAYC,EAAuBC,EAAqB,CAVxD,KAAQ,WAAa,EACrB,KAAQ,kBACR,KAAQ,sBACR,KAAQ,OAAiC,WACzC,KAAQ,YAAc,GACtB,KAAQ,WAAkC,KAMxC,KAAK,QAAU,IAAI,OAAKD,CAAa,EACrC,KAAK,OAASC,EACd,KAAK,QAAQ,OAAO,CAClB,QAASC,EACT,OAAQ,CACN,eAAgB,GAChB,aAAc,GACd,WAAY,EACd,EACA,UAAW,CACT,gBAAiB,GAAGA,CAAkB,gBACtC,qBAAsB,GAAGA,CAAkB,oBAC7C,CACF,CAAC,EACD,KAAK,OAAS,SAAS,cAAc,QAAQ,EAC7C,KAAK,QAAQ,QAAQC,GAAO,EAEtB,CAACA,EAAI,MAAQA,EAAI,OAAS,UAC5B,QAAQ,MAAM,qBAAsBA,CAAG,CAE3C,CAAC,EACD,KAAK,QAAQ,MAAM,EACnB,KAAK,QAAQ,QAAU,IAAM,CA3CjC,IAAAC,EA4CU,KAAK,UACP,KAAK,YAAc,IACnBA,EAAA,KAAK,SAAL,MAAAA,EAAA,WACA,KAAK,QAAQ,IAAI,EACjB,KAAK,QAAQ,qBAAqB,MAAM,EACxC,KAAK,QAAQ,sBAAsB,KAAK,MAAM,EAC9C,KAAK,YAAY,EAErB,CACF,CAEA,SAAkB,CAChB,MAAO,YACT,CAEQ,iBAAiBC,EAAsB,CACzC,KAAK,aACPA,EAAS,EAGP,KAAK,aAAe,MACtB,cAAc,KAAK,UAAU,EAE/B,KAAK,WAAa,YAAY,IAAM,CAC9B,KAAK,cACP,cAAc,KAAK,UAAW,EAC9BA,EAAS,EAEb,EAAG,GAAG,CACR,CAEA,YAAa,CACX,KAAK,WAAa,EAClB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,UAAU,CACzB,CAAC,CACH,CAEA,kBAAmB,CACjB,KAAK,WAAa,GAClB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,gBAAgB,CAC/B,CAAC,CACH,CAKA,QAAQC,EAAc,CACpB,KAAK,WAAaA,EAClB,KAAK,sBACL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,QAAQ,KAAK,UAAU,CACtC,CAAC,CACH,CAKM,UAAUC,EAAgC,QAAAC,EAAA,sBAC9C,YAAK,OAASD,EACP,IAAI,QAAQ,CAACE,EAASC,IAAW,CACtC,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,sBAAsB,KAAK,MAAM,EAAE,KAAKD,CAAO,EAAE,MAAMC,CAAM,CAC5E,CAAC,CACH,CAAC,CACH,GAEA,mBAAmBL,EAAmD,CACpE,KAAK,2BAA6BA,CACpC,CAEA,WAAY,CACV,OAAO,KAAK,MACd,CAEA,eAAgB,CACd,KAAK,sBACL,KAAK,iBAAiB,EACtB,KAAK,WAAW,CAClB,CAEA,cAAcM,EAA2B,CACvC,KAAK,WAAaA,EAClB,KAAK,uBACL,KAAK,WAAW,EAChB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,cAAc,KAAK,UAAU,CAC5C,CAAC,CACH,CAEA,eAAgB,CACd,OAAO,KAAK,UACd,CAEA,eAAgB,CACd,OAAO,KAAK,YAAc,KAAK,cACjC,CAEQ,aAAaC,EAAqB,CACxC,GAAM,CAAE,OAAAC,EAAQ,MAAAC,CAAM,EAAIF,EAAO,eAAe,EAAE,CAAC,EAAE,YAAY,EACjE,KAAK,OAAO,MAAQE,EACpB,KAAK,OAAO,OAASD,EACrB,KAAK,QAAQ,UAAUD,CAAM,EAC7B,KAAK,QAAQ,SAAS,KAAK,MAAM,CACnC,CAEA,MAAMA,EAAkC,CACtC,YAAK,QAAQ,MAAM,EACnB,KAAK,YAAY,EACjB,KAAK,QAAQ,wBAAwB,IAAM,CA3J/C,IAAAR,EA4JM,KAAK,aAAaQ,CAAM,EACxB,GAAM,CAAE,OAAAC,EAAQ,MAAAC,CAAM,EAAIF,EAAO,eAAe,EAAE,CAAC,EAAE,YAAY,GACjER,EAAA,KAAK,6BAAL,MAAAA,EAAA,UAAkCU,EAAQD,EAC5C,CAAC,EACD,KAAK,aAAaD,CAAM,EACjB,KAAK,OAAO,cAAc,EAAE,GAAKA,CAC1C,CAEA,MAAO,CACL,KAAK,cAAc,EACnB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,KAAK,CACpB,CAAC,CACH,CAEQ,aAAc,CAChB,KAAK,WACP,KAAK,QAAQ,KAAK,UAAU,EACnB,KAAK,YACd,KAAK,cAAc,KAAK,UAAU,CAEtC,CACF",
   "names": ["HMSEffectsPlugin_exports", "__export", "HMSEffectsPlugin", "__toCommonJS", "import_effects_sdk", "EFFECTS_SDK_ASSETS", "HMSEffectsPlugin", "effectsSDKKey", "onInit", "EFFECTS_SDK_ASSETS", "err", "_a", "callback", "blur", "preset", "__async", "resolve", "reject", "url", "stream", "height", "width"]
 }

package/dist/cjs/index.js

@@ -1,2 +1,2 @@
-"use strict";var P=Object.defineProperty;var J=Object.getOwnPropertyDescriptor;var K=Object.getOwnPropertyNames;var Q=Object.prototype.hasOwnProperty;var Y=(a,t)=>()=>(t||a((t={exports:{}}).exports,t),t.exports),X=(a,t)=>{for(var e in t)P(a,e,{get:t[e],enumerable:!0})},Z=(a,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of K(t))!Q.call(a,s)&&s!==e&&P(a,s,{get:()=>t[s],enumerable:!(i=J(t,s))||i.enumerable});return a};var tt=a=>Z(P({},"__esModule",{value:!0}),a);var o=(a,t,e)=>new Promise((i,s)=>{var r=h=>{try{u(e.next(h))}catch(d){s(d)}},n=h=>{try{u(e.throw(h))}catch(d){s(d)}},u=h=>h.done?i(h.value):Promise.resolve(h.value).then(r,n);u((e=e.apply(a,t)).next())});var L=Y((mt,et)=>{et.exports={version:"1.13.25-alpha.0",license:"MIT",name:"@100mslive/hms-virtual-background",author:"100ms",module:"dist/esm/index.js",main:"dist/cjs/index.js",typings:"dist/index.d.ts",typesVersions:{"*":{hmsvbplugin:["./dist/HMSVBPlugin.d.ts"],hmseffectsplugin:["./dist/HMSEffectsPlugin.d.ts"]}},exports:{".":{import:"./dist/esm/index.js",require:"./dist/cjs/index.js",default:"./dist/esm/index.js",types:"./dist/index.d.ts"},"./hmsvbplugin":{import:"./dist/esm/HMSVBPlugin.js",require:"./dist/cjs/HMSVBPlugin.js",default:"./dist/esm/HMSVBPlugin.js",types:"./dist/HMSVBPlugin.d.ts"},"./hmseffectsplugin":{import:"./dist/esm/HMSEffectsPlugin.js",default:"./dist/esm/HMSEffectsPlugin.js",require:"./dist/cjs/HMSEffectsPlugin.js",types:"./dist/HMSEffectsPlugin.d.ts"}},repository:{type:"git",url:"https://github.com/100mslive/web-sdks.git",directory:"packages/hms-virtual-background"},files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video-store":"0.12.25-alpha.0"},devDependencies:{"@100mslive/hms-video-store":"0.12.25-alpha.0"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","effects-sdk":"3.2.3","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"b9a15a40498feb37cf34dc6fab790cb9dcfcd775"}});var pt={};X(pt,{HMSEffectsPlugin:()=>T,HMSVBPlugin:()=>I,HMSVirtualBackgroundPlugin:()=>D,HMSVirtualBackgroundTypes:()=>p});module.exports=tt(pt);var y=require("gifuct-js"),x=require("@100mslive/hms-video-store"),Ct=require("@tensorflow/tfjs-backend-webgl");var it=L(),A=`https://unpkg.com/${it.name}/src`,C="VBProcessor",st="tflite/tflite.js",at="tflite/tflite-simd.js",nt="models/selfie_segmentation_landscape.tflite",_=a=>new Promise(function(t,e){let i=document.createElement("script");i.src=a,i.onload=t,i.onerror=e,document.head.appendChild(i)}),ot=()=>o(void 0,null,function*(){let a,t=`${A}/${at}`;yield _(t);try{a=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${A}/${st}`,yield _(t),a=yield createTFLiteModule()}return a}),j=()=>o(void 0,null,function*(){let a=`${A}/${nt}`,[t,e]=yield Promise.all([ot(),fetch(a)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var M="VBProcessor",rt=33,ut=L(),ht=214,dt=855,gt=120,lt=720,D=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(M,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return o(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(M,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=j(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=x.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return ut.name}getPluginType(){return x.HMSVideoPluginType.TRANSFORM}setBackground(t){return o(this,null,function*(){if(t!=="")if(t==="none")this.log(M,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(M,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>o(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>o(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,y.parseGIF)(e)).then(e=>(0,y.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,r),u=Math.exp(s-n),h=Math.exp(r-n);this.personMask.data[i*4+3]=255*h/(u+h)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>ht&&this.output.height>gt&&this.output.width<dt&&this.output.height<lt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/rt?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return o(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,n,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,n=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return o(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),n=t.data,u=t.width,h=t.height,d=u,F=h,R=this.filters.createImageData(d,F),m=R.data,$=i?1:0;for(let g=0;g<F;g=g+1)for(let l=0;l<d;l=l+1){let c=(g*d+l)*4;if(n[c+3]!==0&&l<d&&g<F){let q=g,z=l,O=0,N=0,G=0,H=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let B=q+f-r,V=z+k-r;if(B>=0&&B<h&&V>=0&&V<u){let b=(B*u+V)*4,v=e[f*s+k];O+=n[b]*v,N+=n[b+1]*v,G+=n[b+2]*v,H+=n[b+3]*v}}m[c]=O,m[c+1]=N,m[c+2]=G,m[c+3]=H+$*(255-H)}}return R}}};var U=require("@mediapipe/selfie_segmentation"),w=require("gifuct-js"),S=require("@100mslive/hms-video-store");var p=(n=>(n.BLUR="blur",n.NONE="none",n.GIF="gif",n.IMAGE="image",n.VIDEO="video",n.CANVAS="canvas",n))(p||{});var I=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.background="none";this.backgroundType="none";this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case"image":case"canvas":case"video":this.renderBackground(t,this.background);break;case"gif":this.renderGIF(t);break;case"blur":this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType==="none"||this.backgroundType==="blur")return;this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-out",this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}isBlurSupported(){return"filter"in CanvasRenderingContext2D.prototype}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge","Safari"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=S.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return S.HMSVideoPluginType.TRANSFORM}init(){return o(this,null,function*(){this.segmentation||(this.segmentation=new U.SelfieSegmentation({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return o(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case"none":case"blur":this.background=t,this.backgroundType=e;break;case"image":this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType="image";break;case"video":this.log("setting background to video",t),this.backgroundType="none",this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.playsInline=!0,this.background.oncanplaythrough=()=>o(this,null,function*(){if(this.background&&this.background instanceof HTMLVideoElement)try{yield this.background.play(),this.backgroundType="video"}catch(s){this.log("failed to play background",t)}});break;case"canvas":this.background=t,this.backgroundType="canvas";break;case"gif":if(this.log("setting gif to background",t),this.backgroundType="none",this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif";else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${p}`)}})}getBackground(){return this.background}stop(){var t;this.backgroundType!=="blur"&&this.background!=="none"&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background="none",this.backgroundType="none"}processVideoFrame(t,e,i){return o(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType==="none"){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,w.parseGIF)(e)).then(e=>(0,w.decompressFrames)(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!=="blur"||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!=="gif")){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};var W=require("effects-sdk");var E="https://assets.100ms.live/effectsdk/3.2.3/";var T=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new W.tsvb(t),this.onInit=e,this.effects.config({sdk_url:E,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${E}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${E}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return o(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};
+"use strict";var P=Object.defineProperty;var J=Object.getOwnPropertyDescriptor;var K=Object.getOwnPropertyNames;var Q=Object.prototype.hasOwnProperty;var Y=(a,t)=>()=>(t||a((t={exports:{}}).exports,t),t.exports),X=(a,t)=>{for(var e in t)P(a,e,{get:t[e],enumerable:!0})},Z=(a,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of K(t))!Q.call(a,s)&&s!==e&&P(a,s,{get:()=>t[s],enumerable:!(i=J(t,s))||i.enumerable});return a};var tt=a=>Z(P({},"__esModule",{value:!0}),a);var o=(a,t,e)=>new Promise((i,s)=>{var r=h=>{try{u(e.next(h))}catch(d){s(d)}},n=h=>{try{u(e.throw(h))}catch(d){s(d)}},u=h=>h.done?i(h.value):Promise.resolve(h.value).then(r,n);u((e=e.apply(a,t)).next())});var L=Y((mt,et)=>{et.exports={version:"1.13.25-alpha.10",license:"MIT",name:"@100mslive/hms-virtual-background",author:"100ms",module:"dist/esm/index.js",main:"dist/cjs/index.js",typings:"dist/index.d.ts",typesVersions:{"*":{hmsvbplugin:["./dist/HMSVBPlugin.d.ts"],hmseffectsplugin:["./dist/HMSEffectsPlugin.d.ts"]}},exports:{".":{import:"./dist/esm/index.js",require:"./dist/cjs/index.js",default:"./dist/esm/index.js",types:"./dist/index.d.ts"},"./hmsvbplugin":{import:"./dist/esm/HMSVBPlugin.js",require:"./dist/cjs/HMSVBPlugin.js",default:"./dist/esm/HMSVBPlugin.js",types:"./dist/HMSVBPlugin.d.ts"},"./hmseffectsplugin":{import:"./dist/esm/HMSEffectsPlugin.js",default:"./dist/esm/HMSEffectsPlugin.js",require:"./dist/cjs/HMSEffectsPlugin.js",types:"./dist/HMSEffectsPlugin.d.ts"}},repository:{type:"git",url:"https://github.com/100mslive/web-sdks.git",directory:"packages/hms-virtual-background"},files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video-store":"0.12.25-alpha.10"},devDependencies:{"@100mslive/hms-video-store":"0.12.25-alpha.10"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","effects-sdk":"3.4.3","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"b501de9d52ebead7150aaeaeed746c0ed1ac2c6d"}});var pt={};X(pt,{HMSEffectsPlugin:()=>T,HMSVBPlugin:()=>I,HMSVirtualBackgroundPlugin:()=>D,HMSVirtualBackgroundTypes:()=>p});module.exports=tt(pt);var y=require("gifuct-js"),x=require("@100mslive/hms-video-store"),Ct=require("@tensorflow/tfjs-backend-webgl");var it=L(),A=`https://unpkg.com/${it.name}/src`,C="VBProcessor",st="tflite/tflite.js",at="tflite/tflite-simd.js",nt="models/selfie_segmentation_landscape.tflite",_=a=>new Promise(function(t,e){let i=document.createElement("script");i.src=a,i.onload=t,i.onerror=e,document.head.appendChild(i)}),ot=()=>o(void 0,null,function*(){let a,t=`${A}/${at}`;yield _(t);try{a=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${A}/${st}`,yield _(t),a=yield createTFLiteModule()}return a}),j=()=>o(void 0,null,function*(){let a=`${A}/${nt}`,[t,e]=yield Promise.all([ot(),fetch(a)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var M="VBProcessor",rt=33,ut=L(),ht=214,dt=855,gt=120,lt=720,D=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(M,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return o(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(M,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=j(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=x.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return ut.name}getPluginType(){return x.HMSVideoPluginType.TRANSFORM}setBackground(t){return o(this,null,function*(){if(t!=="")if(t==="none")this.log(M,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(M,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>o(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>o(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,y.parseGIF)(e)).then(e=>(0,y.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,r),u=Math.exp(s-n),h=Math.exp(r-n);this.personMask.data[i*4+3]=255*h/(u+h)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>ht&&this.output.height>gt&&this.output.width<dt&&this.output.height<lt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/rt?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return o(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,n,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,n=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return o(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),n=t.data,u=t.width,h=t.height,d=u,F=h,R=this.filters.createImageData(d,F),m=R.data,$=i?1:0;for(let g=0;g<F;g=g+1)for(let l=0;l<d;l=l+1){let c=(g*d+l)*4;if(n[c+3]!==0&&l<d&&g<F){let q=g,z=l,O=0,N=0,G=0,H=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let B=q+f-r,V=z+k-r;if(B>=0&&B<h&&V>=0&&V<u){let b=(B*u+V)*4,v=e[f*s+k];O+=n[b]*v,N+=n[b+1]*v,G+=n[b+2]*v,H+=n[b+3]*v}}m[c]=O,m[c+1]=N,m[c+2]=G,m[c+3]=H+$*(255-H)}}return R}}};var U=require("@mediapipe/selfie_segmentation"),w=require("gifuct-js"),S=require("@100mslive/hms-video-store");var p=(n=>(n.BLUR="blur",n.NONE="none",n.GIF="gif",n.IMAGE="image",n.VIDEO="video",n.CANVAS="canvas",n))(p||{});var I=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.background="none";this.backgroundType="none";this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case"image":case"canvas":case"video":this.renderBackground(t,this.background);break;case"gif":this.renderGIF(t);break;case"blur":this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType==="none"||this.backgroundType==="blur")return;this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-out",this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}isBlurSupported(){return"filter"in CanvasRenderingContext2D.prototype}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge","Safari"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=S.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return S.HMSVideoPluginType.TRANSFORM}init(){return o(this,null,function*(){this.segmentation||(this.segmentation=new U.SelfieSegmentation({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return o(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case"none":case"blur":this.background=t,this.backgroundType=e;break;case"image":this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType="image";break;case"video":this.log("setting background to video",t),this.backgroundType="none",this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.playsInline=!0,this.background.oncanplaythrough=()=>o(this,null,function*(){if(this.background&&this.background instanceof HTMLVideoElement)try{yield this.background.play(),this.backgroundType="video"}catch(s){this.log("failed to play background",t)}});break;case"canvas":this.background=t,this.backgroundType="canvas";break;case"gif":if(this.log("setting gif to background",t),this.backgroundType="none",this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif";else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${p}`)}})}getBackground(){return this.background}stop(){var t;this.backgroundType!=="blur"&&this.background!=="none"&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background="none",this.backgroundType="none"}processVideoFrame(t,e,i){return o(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType==="none"){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,w.parseGIF)(e)).then(e=>(0,w.decompressFrames)(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!=="blur"||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!=="gif")){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};var W=require("effects-sdk");var E="https://assets.100ms.live/effectsdk/3.4.3/";var T=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new W.tsvb(t),this.onInit=e,this.effects.config({sdk_url:E,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${E}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${E}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return o(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};
 //# sourceMappingURL=index.js.map