@100mslive/hms-virtual-background 1.9.2-alpha.2 → 1.9.2-alpha.4

package/README.md

@@ -1,27 +1,13 @@
-# How to use
+## installation
 
-1. npm i @100mslive/hms-virtual-background
-2. import { setVirtualBackground } from '@100mslive/hms-virtual-background'
-3. setVirtualBackground accepts 2 parameters :-
-   a) background, Its a string, can be one of these 3 parameters: ['default' | 'blur' | 'Image']
-   i) 'default' -> No virtual background is set
-   ii) 'blur' -> Background will be blurred
-   iii) 'Image' -> It's a source URL of image you want to replace your background with
-   b) stream: It's the input camera feed on which virtual background will work
+**with npm:**
 
-   For ex:->
-   To blur your background, Use: `setVirtualBackground( background: 'blur', stream: stream )`
-   To Add Virtual Image, specify an Image URL in background, Use: setVirtualBackground(background: Image, stream: stream)
-   For default mode , Use `setVirtualBackground( background: 'default', stream: stream )`
+```npm i --save @100mslive/hms-virtual-background```
 
-## Commands
+**with yarn**
 
-```bash
-npm start # or yarn start
-```
+```yarn add @100mslive/hms-virtual-background```
 
-This builds to `/dist` and runs the project in watch mode so any edits you save inside `src` causes a rebuild to `/dist`.
+## Usage
 
-To do a one-off build, use `npm run build` or `yarn build`.
-
-To run tests, use `npm test` or `yarn test`.
+Refer our [docs](https://www.100ms.live/docs/javascript/v2/plugins/virtual-background#start-and-stop-virtual-background) for usage.

package/dist/index.cjs.js

@@ -1,2 +1,2 @@
-var q=Object.create;var x=Object.defineProperty;var J=Object.getOwnPropertyDescriptor;var Q=Object.getOwnPropertyNames;var Y=Object.getPrototypeOf,K=Object.prototype.hasOwnProperty;var A=n=>x(n,"__esModule",{value:!0});var X=(n,t)=>()=>(t||n((t={exports:{}}).exports,t),t.exports),Z=(n,t)=>{A(n);for(var e in t)x(n,e,{get:t[e],enumerable:!0})},tt=(n,t,e)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of Q(t))!K.call(n,i)&&i!=="default"&&x(n,i,{get:()=>t[i],enumerable:!(e=J(t,i))||e.enumerable});return n},c=n=>tt(A(x(n!=null?q(Y(n)):{},"default",n&&n.__esModule&&"default"in n?{get:()=>n.default,enumerable:!0}:{value:n,enumerable:!0})),n);var r=(n,t,e)=>new Promise((i,s)=>{var h=g=>{try{u(e.next(g))}catch(d){s(d)}},o=g=>{try{u(e.throw(g))}catch(d){s(d)}},u=g=>g.done?i(g.value):Promise.resolve(g.value).then(h,o);u((e=e.apply(n,t)).next())});var V=X((ct,et)=>{et.exports={version:"1.9.2-alpha.2",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.5.2"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.7.2-alpha.2"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"7b2b9bc338ae9edc003ef618ed5ff2867a08ce83"}});Z(exports,{HMSVBPlugin:()=>U,HMSVirtualBackgroundPlugin:()=>_,HMSVirtualBackgroundTypes:()=>a});var I=c(require("gifuct-js")),y=c(require("@100mslive/hms-video")),bt=c(require("@tensorflow/tfjs-backend-webgl"));var it=V(),P=`https://unpkg.com/${it.name}/src`,M="VBProcessor",st="tflite/tflite.js",at="tflite/tflite-simd.js",nt="models/selfie_segmentation_landscape.tflite",G=n=>new Promise(function(t,e){let i=document.createElement("script");i.src=n,i.onload=t,i.onerror=e,document.head.appendChild(i)}),ot=()=>r(void 0,null,function*(){let n,t=`${P}/${at}`;yield G(t);try{n=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${P}/${st}`,yield G(t),n=yield createTFLiteModule()}return n}),N=()=>r(void 0,null,function*(){let n=`${P}/${nt}`,[t,e]=yield Promise.all([ot(),fetch(n)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(M,"Input memory offset:",t._getInputMemoryOffset()),console.debug(M,"Input height:",t._getInputHeight()),console.debug(M,"Input width:",t._getInputWidth()),console.debug(M,"Input channels:",t._getInputChannelCount()),t});var w="VBProcessor",rt=33,ht=V(),ut=214,gt=855,dt=120,lt=720,_=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(w,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return r(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(w,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=N(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=y.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return ht.name}getPluginType(){return y.HMSVideoPluginType.TRANSFORM}setBackground(t){return r(this,null,function*(){if(t!=="")if(t==="none")this.log(w,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(w,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>r(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let h=()=>r(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):h()}setImage(t){return r(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,I.parseGIF)(e)).then(e=>(0,I.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],h=this.tfLite.HEAPF32[e+i*2+1],o=Math.max(s,h),u=Math.exp(s-o),g=Math.exp(h-o);this.personMask.data[i*4+3]=255*g/(u+g)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>ut&&this.output.height>dt&&this.output.width<gt&&this.output.height<lt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/rt?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return r(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,h,o,u;e/i<this.imageAspectRatio?(s=e,h=e/this.imageAspectRatio,o=0,u=(i-h)/2):(h=i,s=i*this.imageAspectRatio,u=0,o=(e-s)/2),this.outputCtx.drawImage(t,o,u,s,h,0,0,this.output.width,this.output.height)}addBlurToBackground(){return r(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),h=Math.floor(s/2),o=t.data,u=t.width,g=t.height,d=u,E=g,D=this.filters.createImageData(d,E),f=D.data,W=i?1:0;for(let l=0;l<E;l=l+1)for(let p=0;p<d;p=p+1){let m=(l*d+p)*4;if(o[m+3]!==0&&p<d&&l<E){let $=l,z=p,B=0,O=0,R=0,S=0;for(let k=0;k<s;k++)for(let b=0;b<s;b++){let L=$+k-h,H=z+b-h;if(L>=0&&L<g&&H>=0&&H<u){let v=(L*u+H)*4,C=e[k*s+b];B+=o[v]*C,O+=o[v+1]*C,R+=o[v+2]*C,S+=o[v+3]*C}}f[m]=B,f[m+1]=O,f[m+2]=R,f[m+3]=S+W*(255-S)}}return D}}};var j=c(require("@mediapipe/selfie_segmentation")),F=c(require("gifuct-js")),T=c(require("@100mslive/hms-video"));var a;(function(o){o.BLUR="blur",o.NONE="none",o.GIF="gif",o.IMAGE="image",o.VIDEO="video",o.CANVAS="canvas"})(a||(a={}));var U=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.backgroundType=a.NONE;this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case a.IMAGE:case a.CANVAS:case a.VIDEO:this.renderBackground(t,this.background);break;case a.GIF:this.renderGIF(t);break;case a.BLUR:this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType===a.NONE||this.backgroundType===a.BLUR)return;this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=T.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return T.HMSVideoPluginType.TRANSFORM}init(){return r(this,null,function*(){this.segmentation||(this.segmentation=new j.SelfieSegmentation({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return r(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case a.NONE:case a.BLUR:this.background=t,this.backgroundType=e;break;case a.IMAGE:this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType=a.IMAGE;break;case a.VIDEO:this.log("setting background to video",t),this.backgroundType=a.NONE,this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.autoplay=!0,this.background.oncanplaythrough=()=>{this.backgroundType=a.VIDEO};break;case a.CANVAS:this.background=t,this.backgroundType=a.CANVAS;break;case a.GIF:if(this.log("setting gif to background",t),this.backgroundType=a.NONE,this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType=a.GIF;else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${a}`)}})}stop(){var t;this.backgroundType!==a.BLUR&&this.background!==a.NONE&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background=a.NONE,this.backgroundType=a.NONE}processVideoFrame(t,e,i){return r(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType===a.NONE){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return r(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,F.parseGIF)(e)).then(e=>(0,F.decompressFrames)(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!==a.BLUR||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!==a.GIF)){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};
+var q=Object.create;var x=Object.defineProperty;var J=Object.getOwnPropertyDescriptor;var Q=Object.getOwnPropertyNames;var Y=Object.getPrototypeOf,K=Object.prototype.hasOwnProperty;var A=n=>x(n,"__esModule",{value:!0});var X=(n,t)=>()=>(t||n((t={exports:{}}).exports,t),t.exports),Z=(n,t)=>{A(n);for(var e in t)x(n,e,{get:t[e],enumerable:!0})},tt=(n,t,e)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of Q(t))!K.call(n,i)&&i!=="default"&&x(n,i,{get:()=>t[i],enumerable:!(e=J(t,i))||e.enumerable});return n},c=n=>tt(A(x(n!=null?q(Y(n)):{},"default",n&&n.__esModule&&"default"in n?{get:()=>n.default,enumerable:!0}:{value:n,enumerable:!0})),n);var r=(n,t,e)=>new Promise((i,s)=>{var h=g=>{try{u(e.next(g))}catch(d){s(d)}},o=g=>{try{u(e.throw(g))}catch(d){s(d)}},u=g=>g.done?i(g.value):Promise.resolve(g.value).then(h,o);u((e=e.apply(n,t)).next())});var V=X((ct,et)=>{et.exports={version:"1.9.2-alpha.4",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.7.1"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.7.2-alpha.4"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"0f151aa3c19ee21af49e0f5de4600e0b9caf61d1"}});Z(exports,{HMSVBPlugin:()=>U,HMSVirtualBackgroundPlugin:()=>_,HMSVirtualBackgroundTypes:()=>a});var I=c(require("gifuct-js")),y=c(require("@100mslive/hms-video")),bt=c(require("@tensorflow/tfjs-backend-webgl"));var it=V(),P=`https://unpkg.com/${it.name}/src`,M="VBProcessor",st="tflite/tflite.js",at="tflite/tflite-simd.js",nt="models/selfie_segmentation_landscape.tflite",G=n=>new Promise(function(t,e){let i=document.createElement("script");i.src=n,i.onload=t,i.onerror=e,document.head.appendChild(i)}),ot=()=>r(void 0,null,function*(){let n,t=`${P}/${at}`;yield G(t);try{n=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${P}/${st}`,yield G(t),n=yield createTFLiteModule()}return n}),N=()=>r(void 0,null,function*(){let n=`${P}/${nt}`,[t,e]=yield Promise.all([ot(),fetch(n)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(M,"Input memory offset:",t._getInputMemoryOffset()),console.debug(M,"Input height:",t._getInputHeight()),console.debug(M,"Input width:",t._getInputWidth()),console.debug(M,"Input channels:",t._getInputChannelCount()),t});var w="VBProcessor",rt=33,ht=V(),ut=214,gt=855,dt=120,lt=720,_=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(w,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return r(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(w,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=N(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=y.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return ht.name}getPluginType(){return y.HMSVideoPluginType.TRANSFORM}setBackground(t){return r(this,null,function*(){if(t!=="")if(t==="none")this.log(w,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(w,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>r(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let h=()=>r(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):h()}setImage(t){return r(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,I.parseGIF)(e)).then(e=>(0,I.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],h=this.tfLite.HEAPF32[e+i*2+1],o=Math.max(s,h),u=Math.exp(s-o),g=Math.exp(h-o);this.personMask.data[i*4+3]=255*g/(u+g)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>ut&&this.output.height>dt&&this.output.width<gt&&this.output.height<lt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/rt?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return r(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,h,o,u;e/i<this.imageAspectRatio?(s=e,h=e/this.imageAspectRatio,o=0,u=(i-h)/2):(h=i,s=i*this.imageAspectRatio,u=0,o=(e-s)/2),this.outputCtx.drawImage(t,o,u,s,h,0,0,this.output.width,this.output.height)}addBlurToBackground(){return r(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),h=Math.floor(s/2),o=t.data,u=t.width,g=t.height,d=u,E=g,D=this.filters.createImageData(d,E),f=D.data,W=i?1:0;for(let l=0;l<E;l=l+1)for(let p=0;p<d;p=p+1){let m=(l*d+p)*4;if(o[m+3]!==0&&p<d&&l<E){let $=l,z=p,B=0,O=0,R=0,S=0;for(let k=0;k<s;k++)for(let b=0;b<s;b++){let L=$+k-h,H=z+b-h;if(L>=0&&L<g&&H>=0&&H<u){let v=(L*u+H)*4,C=e[k*s+b];B+=o[v]*C,O+=o[v+1]*C,R+=o[v+2]*C,S+=o[v+3]*C}}f[m]=B,f[m+1]=O,f[m+2]=R,f[m+3]=S+W*(255-S)}}return D}}};var j=c(require("@mediapipe/selfie_segmentation")),F=c(require("gifuct-js")),T=c(require("@100mslive/hms-video"));var a;(function(o){o.BLUR="blur",o.NONE="none",o.GIF="gif",o.IMAGE="image",o.VIDEO="video",o.CANVAS="canvas"})(a||(a={}));var U=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.backgroundType=a.NONE;this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case a.IMAGE:case a.CANVAS:case a.VIDEO:this.renderBackground(t,this.background);break;case a.GIF:this.renderGIF(t);break;case a.BLUR:this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType===a.NONE||this.backgroundType===a.BLUR)return;this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=T.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return T.HMSVideoPluginType.TRANSFORM}init(){return r(this,null,function*(){this.segmentation||(this.segmentation=new j.SelfieSegmentation({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return r(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case a.NONE:case a.BLUR:this.background=t,this.backgroundType=e;break;case a.IMAGE:this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType=a.IMAGE;break;case a.VIDEO:this.log("setting background to video",t),this.backgroundType=a.NONE,this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.autoplay=!0,this.background.oncanplaythrough=()=>{this.backgroundType=a.VIDEO};break;case a.CANVAS:this.background=t,this.backgroundType=a.CANVAS;break;case a.GIF:if(this.log("setting gif to background",t),this.backgroundType=a.NONE,this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType=a.GIF;else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${a}`)}})}stop(){var t;this.backgroundType!==a.BLUR&&this.background!==a.NONE&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background=a.NONE,this.backgroundType=a.NONE}processVideoFrame(t,e,i){return r(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType===a.NONE){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return r(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,F.parseGIF)(e)).then(e=>(0,F.decompressFrames)(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!==a.BLUR||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!==a.GIF)){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};
 //# sourceMappingURL=index.cjs.js.map

package/dist/index.js

@@ -1,2 +1,2 @@
-var R=(h,t)=>()=>(t||h((t={exports:{}}).exports,t),t.exports);var o=(h,t,e)=>new Promise((i,s)=>{var r=g=>{try{u(e.next(g))}catch(d){s(d)}},n=g=>{try{u(e.throw(g))}catch(d){s(d)}},u=g=>g.done?i(g.value):Promise.resolve(g.value).then(r,n);u((e=e.apply(h,t)).next())});var F=R((ht,A)=>{A.exports={version:"1.9.2-alpha.2",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.5.2"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.7.2-alpha.2"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"7b2b9bc338ae9edc003ef618ed5ff2867a08ce83"}});import{decompressFrames as W,parseGIF as $}from"gifuct-js";import{HMSPluginUnsupportedTypes as z,HMSVideoPluginType as q}from"@100mslive/hms-video";import"@tensorflow/tfjs-backend-webgl";var G=F(),T=`https://unpkg.com/${G.name}/src`,C="VBProcessor",N="tflite/tflite.js",_="tflite/tflite-simd.js",j="models/selfie_segmentation_landscape.tflite",V=h=>new Promise(function(t,e){let i=document.createElement("script");i.src=h,i.onload=t,i.onerror=e,document.head.appendChild(i)}),U=()=>o(void 0,null,function*(){let h,t=`${T}/${_}`;yield V(t);try{h=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${T}/${N}`,yield V(t),h=yield createTFLiteModule()}return h}),P=()=>o(void 0,null,function*(){let h=`${T}/${j}`,[t,e]=yield Promise.all([U(),fetch(h)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var x="VBProcessor",J=33,Q=F(),Y=214,K=855,X=120,Z=720,tt=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(x,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return o(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(x,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=P(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=z.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return Q.name}getPluginType(){return q.TRANSFORM}setBackground(t){return o(this,null,function*(){if(t!=="")if(t==="none")this.log(x,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(x,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>o(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>o(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>$(e)).then(e=>W(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,r),u=Math.exp(s-n),g=Math.exp(r-n);this.personMask.data[i*4+3]=255*g/(u+g)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>Y&&this.output.height>X&&this.output.width<K&&this.output.height<Z&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/J?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return o(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,n,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,n=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return o(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),n=t.data,u=t.width,g=t.height,d=u,M=g,E=this.filters.createImageData(d,M),m=E.data,D=i?1:0;for(let l=0;l<M;l=l+1)for(let p=0;p<d;p=p+1){let c=(l*d+p)*4;if(n[c+3]!==0&&p<d&&l<M){let B=l,O=p,S=0,L=0,H=0,I=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let y=B+f-r,w=O+k-r;if(y>=0&&y<g&&w>=0&&w<u){let b=(y*u+w)*4,v=e[f*s+k];S+=n[b]*v,L+=n[b+1]*v,H+=n[b+2]*v,I+=n[b+3]*v}}m[c]=S,m[c+1]=L,m[c+2]=H,m[c+3]=I+D*(255-I)}}return E}}};import{SelfieSegmentation as et}from"@mediapipe/selfie_segmentation";import{decompressFrames as it,parseGIF as st}from"gifuct-js";import{HMSPluginUnsupportedTypes as at,HMSVideoPluginType as nt}from"@100mslive/hms-video";var a;(function(n){n.BLUR="blur",n.NONE="none",n.GIF="gif",n.IMAGE="image",n.VIDEO="video",n.CANVAS="canvas"})(a||(a={}));var ot=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.backgroundType=a.NONE;this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case a.IMAGE:case a.CANVAS:case a.VIDEO:this.renderBackground(t,this.background);break;case a.GIF:this.renderGIF(t);break;case a.BLUR:this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType===a.NONE||this.backgroundType===a.BLUR)return;this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=at.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return nt.TRANSFORM}init(){return o(this,null,function*(){this.segmentation||(this.segmentation=new et({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return o(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case a.NONE:case a.BLUR:this.background=t,this.backgroundType=e;break;case a.IMAGE:this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType=a.IMAGE;break;case a.VIDEO:this.log("setting background to video",t),this.backgroundType=a.NONE,this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.autoplay=!0,this.background.oncanplaythrough=()=>{this.backgroundType=a.VIDEO};break;case a.CANVAS:this.background=t,this.backgroundType=a.CANVAS;break;case a.GIF:if(this.log("setting gif to background",t),this.backgroundType=a.NONE,this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType=a.GIF;else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${a}`)}})}stop(){var t;this.backgroundType!==a.BLUR&&this.background!==a.NONE&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background=a.NONE,this.backgroundType=a.NONE}processVideoFrame(t,e,i){return o(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType===a.NONE){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>st(e)).then(e=>it(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!==a.BLUR||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!==a.GIF)){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};export{ot as HMSVBPlugin,tt as HMSVirtualBackgroundPlugin,a as HMSVirtualBackgroundTypes};
+var R=(h,t)=>()=>(t||h((t={exports:{}}).exports,t),t.exports);var o=(h,t,e)=>new Promise((i,s)=>{var r=g=>{try{u(e.next(g))}catch(d){s(d)}},n=g=>{try{u(e.throw(g))}catch(d){s(d)}},u=g=>g.done?i(g.value):Promise.resolve(g.value).then(r,n);u((e=e.apply(h,t)).next())});var F=R((ht,A)=>{A.exports={version:"1.9.2-alpha.4",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.7.1"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.7.2-alpha.4"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"0f151aa3c19ee21af49e0f5de4600e0b9caf61d1"}});import{decompressFrames as W,parseGIF as $}from"gifuct-js";import{HMSPluginUnsupportedTypes as z,HMSVideoPluginType as q}from"@100mslive/hms-video";import"@tensorflow/tfjs-backend-webgl";var G=F(),T=`https://unpkg.com/${G.name}/src`,C="VBProcessor",N="tflite/tflite.js",_="tflite/tflite-simd.js",j="models/selfie_segmentation_landscape.tflite",V=h=>new Promise(function(t,e){let i=document.createElement("script");i.src=h,i.onload=t,i.onerror=e,document.head.appendChild(i)}),U=()=>o(void 0,null,function*(){let h,t=`${T}/${_}`;yield V(t);try{h=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${T}/${N}`,yield V(t),h=yield createTFLiteModule()}return h}),P=()=>o(void 0,null,function*(){let h=`${T}/${j}`,[t,e]=yield Promise.all([U(),fetch(h)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var x="VBProcessor",J=33,Q=F(),Y=214,K=855,X=120,Z=720,tt=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(x,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return o(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(x,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=P(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=z.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return Q.name}getPluginType(){return q.TRANSFORM}setBackground(t){return o(this,null,function*(){if(t!=="")if(t==="none")this.log(x,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(x,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>o(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>o(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>$(e)).then(e=>W(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,r),u=Math.exp(s-n),g=Math.exp(r-n);this.personMask.data[i*4+3]=255*g/(u+g)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>Y&&this.output.height>X&&this.output.width<K&&this.output.height<Z&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/J?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return o(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,n,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,n=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return o(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),n=t.data,u=t.width,g=t.height,d=u,M=g,E=this.filters.createImageData(d,M),m=E.data,D=i?1:0;for(let l=0;l<M;l=l+1)for(let p=0;p<d;p=p+1){let c=(l*d+p)*4;if(n[c+3]!==0&&p<d&&l<M){let B=l,O=p,S=0,L=0,H=0,I=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let y=B+f-r,w=O+k-r;if(y>=0&&y<g&&w>=0&&w<u){let b=(y*u+w)*4,v=e[f*s+k];S+=n[b]*v,L+=n[b+1]*v,H+=n[b+2]*v,I+=n[b+3]*v}}m[c]=S,m[c+1]=L,m[c+2]=H,m[c+3]=I+D*(255-I)}}return E}}};import{SelfieSegmentation as et}from"@mediapipe/selfie_segmentation";import{decompressFrames as it,parseGIF as st}from"gifuct-js";import{HMSPluginUnsupportedTypes as at,HMSVideoPluginType as nt}from"@100mslive/hms-video";var a;(function(n){n.BLUR="blur",n.NONE="none",n.GIF="gif",n.IMAGE="image",n.VIDEO="video",n.CANVAS="canvas"})(a||(a={}));var ot=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.backgroundType=a.NONE;this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case a.IMAGE:case a.CANVAS:case a.VIDEO:this.renderBackground(t,this.background);break;case a.GIF:this.renderGIF(t);break;case a.BLUR:this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType===a.NONE||this.backgroundType===a.BLUR)return;this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=at.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return nt.TRANSFORM}init(){return o(this,null,function*(){this.segmentation||(this.segmentation=new et({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return o(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case a.NONE:case a.BLUR:this.background=t,this.backgroundType=e;break;case a.IMAGE:this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType=a.IMAGE;break;case a.VIDEO:this.log("setting background to video",t),this.backgroundType=a.NONE,this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.autoplay=!0,this.background.oncanplaythrough=()=>{this.backgroundType=a.VIDEO};break;case a.CANVAS:this.background=t,this.backgroundType=a.CANVAS;break;case a.GIF:if(this.log("setting gif to background",t),this.backgroundType=a.NONE,this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType=a.GIF;else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${a}`)}})}stop(){var t;this.backgroundType!==a.BLUR&&this.background!==a.NONE&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background=a.NONE,this.backgroundType=a.NONE}processVideoFrame(t,e,i){return o(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType===a.NONE){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>st(e)).then(e=>it(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!==a.BLUR||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!==a.GIF)){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};export{ot as HMSVBPlugin,tt as HMSVirtualBackgroundPlugin,a as HMSVirtualBackgroundTypes};
 //# sourceMappingURL=index.js.map

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.9.2-alpha.2",
+  "version": "1.9.2-alpha.4",
   "license": "MIT",
   "main": "dist/index.cjs.js",
   "typings": "dist/index.d.ts",
@@ -24,13 +24,13 @@
     "format": "prettier --write src/**/*.ts"
   },
   "peerDependencies": {
-    "@100mslive/hms-video": "^0.5.2"
+    "@100mslive/hms-video": "^0.7.1"
   },
   "name": "@100mslive/hms-virtual-background",
   "author": "ashish17",
   "module": "dist/index.js",
   "devDependencies": {
-    "@100mslive/hms-video": "0.7.2-alpha.2"
+    "@100mslive/hms-video": "0.7.2-alpha.4"
   },
   "dependencies": {
     "@mediapipe/selfie_segmentation": "^0.1.1632777926",
@@ -51,5 +51,5 @@
     "defineTFLite.ts",
     "importing.test.ts"
   ],
-  "gitHead": "7b2b9bc338ae9edc003ef618ed5ff2867a08ce83"
+  "gitHead": "0f151aa3c19ee21af49e0f5de4600e0b9caf61d1"
 }