@100mslive/hms-virtual-background 1.8.0 → 1.8.1

package/dist/index.cjs.js

@@ -1,2 +1,2 @@
-var W=Object.create;var M=Object.defineProperty;var N=Object.getOwnPropertyDescriptor;var $=Object.getOwnPropertyNames;var U=Object.getPrototypeOf,z=Object.prototype.hasOwnProperty;var B=s=>M(s,"__esModule",{value:!0});var q=(s,t)=>()=>(t||s((t={exports:{}}).exports,t),t.exports),J=(s,t)=>{B(s);for(var e in t)M(s,e,{get:t[e],enumerable:!0})},Y=(s,t,e)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of $(t))!z.call(s,i)&&i!=="default"&&M(s,i,{get:()=>t[i],enumerable:!(e=N(t,i))||e.enumerable});return s},L=s=>Y(B(M(s!=null?W(U(s)):{},"default",s&&s.__esModule&&"default"in s?{get:()=>s.default,enumerable:!0}:{value:s,enumerable:!0})),s);var u=(s,t,e)=>new Promise((i,a)=>{var n=h=>{try{r(e.next(h))}catch(l){a(l)}},o=h=>{try{r(e.throw(h))}catch(l){a(l)}},r=h=>h.done?i(h.value):Promise.resolve(h.value).then(n,o);r((e=e.apply(s,t)).next())});var E=q((ut,Q)=>{Q.exports={version:"1.8.0",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.5.2"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.6.0"},dependencies:{"@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-core":"^3.3.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"d5cc829f7c2177b8db707f04b8fe95246167fdcd"}});J(exports,{HMSVirtualBackgroundPlugin:()=>_});var x=L(require("gifuct-js")),w=L(require("@100mslive/hms-video")),ct=L(require("@tensorflow/tfjs-backend-webgl"));var K=E(),V=`https://unpkg.com/${K.name}/src`,y="VBProcessor",X="tflite/tflite.js",Z="tflite/tflite-simd.js",tt="models/selfie_segmentation_landscape.tflite",A=s=>new Promise(function(t,e){let i=document.createElement("script");i.src=s,i.onload=t,i.onerror=e,document.head.appendChild(i)}),et=()=>u(void 0,null,function*(){let s,t=`${V}/${Z}`;yield A(t);try{s=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${V}/${X}`,yield A(t),s=yield createTFLiteModule()}return s}),O=()=>u(void 0,null,function*(){let s=`${V}/${tt}`,[t,e]=yield Promise.all([et(),fetch(s)]),i=yield e.arrayBuffer(),a=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),a),t._loadModel(i.byteLength),console.debug(y,"Input memory offset:",t._getInputMemoryOffset()),console.debug(y,"Input height:",t._getInputHeight()),console.debug(y,"Input width:",t._getInputWidth()),console.debug(y,"Input channels:",t._getInputChannelCount()),t});var v="VBProcessor",it=33,st=E(),at=214,nt=855,ot=120,rt=720,_=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(v,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return u(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(v,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=O(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=w.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return st.name}getPluginType(){return w.HMSVideoPluginType.TRANSFORM}setBackground(t){return u(this,null,function*(){if(t!=="")if(t==="none")this.log(v,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(v,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>u(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let a=e.getContext("2d");if(a.canvas.width!==t.width&&(a.canvas.width=t.width),a.canvas.height!==t.height&&(a.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=a,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let n=()=>u(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):n()}setImage(t){return u(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,x.parseGIF)(e)).then(e=>(0,x.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let a=this.tfLite.HEAPF32[e+i*2],n=this.tfLite.HEAPF32[e+i*2+1],o=Math.max(a,n),r=Math.exp(a-o),h=Math.exp(n-o);this.personMask.data[i*4+3]=255*h/(r+h)}else if(this.modelName==="landscape-segmentation"){let a=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*a}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>at&&this.output.height>ot&&this.output.width<nt&&this.output.height<rt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/it?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return u(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let a,n,o,r;e/i<this.imageAspectRatio?(a=e,n=e/this.imageAspectRatio,o=0,r=(i-n)/2):(n=i,a=i*this.imageAspectRatio,r=0,o=(e-a)/2),this.outputCtx.drawImage(t,o,r,a,n,0,0,this.output.width,this.output.height)}addBlurToBackground(){return u(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let a=Math.round(Math.sqrt(e.length)),n=Math.floor(a/2),o=t.data,r=t.width,h=t.height,l=r,I=h,S=this.filters.createImageData(l,I),p=S.data,R=i?1:0;for(let d=0;d<I;d=d+1)for(let g=0;g<l;g=g+1){let c=(d*l+g)*4;if(o[c+3]!==0&&g<l&&d<I){let j=d,G=g,H=0,P=0,D=0,C=0;for(let m=0;m<a;m++)for(let f=0;f<a;f++){let F=j+m-n,T=G+f-n;if(F>=0&&F<h&&T>=0&&T<r){let k=(F*r+T)*4,b=e[m*a+f];H+=o[k]*b,P+=o[k+1]*b,D+=o[k+2]*b,C+=o[k+3]*b}}p[c]=H,p[c+1]=P,p[c+2]=D,p[c+3]=C+R*(255-C)}}return S}}};
+var W=Object.create;var M=Object.defineProperty;var N=Object.getOwnPropertyDescriptor;var $=Object.getOwnPropertyNames;var U=Object.getPrototypeOf,z=Object.prototype.hasOwnProperty;var B=s=>M(s,"__esModule",{value:!0});var q=(s,t)=>()=>(t||s((t={exports:{}}).exports,t),t.exports),J=(s,t)=>{B(s);for(var e in t)M(s,e,{get:t[e],enumerable:!0})},Y=(s,t,e)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of $(t))!z.call(s,i)&&i!=="default"&&M(s,i,{get:()=>t[i],enumerable:!(e=N(t,i))||e.enumerable});return s},L=s=>Y(B(M(s!=null?W(U(s)):{},"default",s&&s.__esModule&&"default"in s?{get:()=>s.default,enumerable:!0}:{value:s,enumerable:!0})),s);var u=(s,t,e)=>new Promise((i,a)=>{var n=h=>{try{r(e.next(h))}catch(l){a(l)}},o=h=>{try{r(e.throw(h))}catch(l){a(l)}},r=h=>h.done?i(h.value):Promise.resolve(h.value).then(n,o);r((e=e.apply(s,t)).next())});var E=q((ut,Q)=>{Q.exports={version:"1.8.1",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.5.2"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.6.1"},dependencies:{"@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-core":"^3.3.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"b4d98c849c3e4ae4e583b2e2c8f9813193cf1d7c"}});J(exports,{HMSVirtualBackgroundPlugin:()=>_});var x=L(require("gifuct-js")),w=L(require("@100mslive/hms-video")),ct=L(require("@tensorflow/tfjs-backend-webgl"));var K=E(),V=`https://unpkg.com/${K.name}/src`,y="VBProcessor",X="tflite/tflite.js",Z="tflite/tflite-simd.js",tt="models/selfie_segmentation_landscape.tflite",A=s=>new Promise(function(t,e){let i=document.createElement("script");i.src=s,i.onload=t,i.onerror=e,document.head.appendChild(i)}),et=()=>u(void 0,null,function*(){let s,t=`${V}/${Z}`;yield A(t);try{s=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${V}/${X}`,yield A(t),s=yield createTFLiteModule()}return s}),O=()=>u(void 0,null,function*(){let s=`${V}/${tt}`,[t,e]=yield Promise.all([et(),fetch(s)]),i=yield e.arrayBuffer(),a=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),a),t._loadModel(i.byteLength),console.debug(y,"Input memory offset:",t._getInputMemoryOffset()),console.debug(y,"Input height:",t._getInputHeight()),console.debug(y,"Input width:",t._getInputWidth()),console.debug(y,"Input channels:",t._getInputChannelCount()),t});var v="VBProcessor",it=33,st=E(),at=214,nt=855,ot=120,rt=720,_=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(v,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return u(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(v,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=O(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=w.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return st.name}getPluginType(){return w.HMSVideoPluginType.TRANSFORM}setBackground(t){return u(this,null,function*(){if(t!=="")if(t==="none")this.log(v,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(v,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>u(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let a=e.getContext("2d");if(a.canvas.width!==t.width&&(a.canvas.width=t.width),a.canvas.height!==t.height&&(a.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=a,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let n=()=>u(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):n()}setImage(t){return u(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,x.parseGIF)(e)).then(e=>(0,x.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let a=this.tfLite.HEAPF32[e+i*2],n=this.tfLite.HEAPF32[e+i*2+1],o=Math.max(a,n),r=Math.exp(a-o),h=Math.exp(n-o);this.personMask.data[i*4+3]=255*h/(r+h)}else if(this.modelName==="landscape-segmentation"){let a=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*a}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>at&&this.output.height>ot&&this.output.width<nt&&this.output.height<rt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/it?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return u(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let a,n,o,r;e/i<this.imageAspectRatio?(a=e,n=e/this.imageAspectRatio,o=0,r=(i-n)/2):(n=i,a=i*this.imageAspectRatio,r=0,o=(e-a)/2),this.outputCtx.drawImage(t,o,r,a,n,0,0,this.output.width,this.output.height)}addBlurToBackground(){return u(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let a=Math.round(Math.sqrt(e.length)),n=Math.floor(a/2),o=t.data,r=t.width,h=t.height,l=r,I=h,S=this.filters.createImageData(l,I),p=S.data,R=i?1:0;for(let d=0;d<I;d=d+1)for(let g=0;g<l;g=g+1){let c=(d*l+g)*4;if(o[c+3]!==0&&g<l&&d<I){let j=d,G=g,H=0,P=0,D=0,C=0;for(let m=0;m<a;m++)for(let f=0;f<a;f++){let F=j+m-n,T=G+f-n;if(F>=0&&F<h&&T>=0&&T<r){let k=(F*r+T)*4,b=e[m*a+f];H+=o[k]*b,P+=o[k+1]*b,D+=o[k+2]*b,C+=o[k+3]*b}}p[c]=H,p[c+1]=P,p[c+2]=D,p[c+3]=C+R*(255-C)}}return S}}};
 //# sourceMappingURL=index.cjs.js.map

package/dist/index.js

@@ -1,2 +1,2 @@
-var A=(h,t)=>()=>(t||h((t={exports:{}}).exports,t),t.exports);var u=(h,t,e)=>new Promise((i,s)=>{var a=r=>{try{o(e.next(r))}catch(l){s(l)}},n=r=>{try{o(e.throw(r))}catch(l){s(l)}},o=r=>r.done?i(r.value):Promise.resolve(r.value).then(a,n);o((e=e.apply(h,t)).next())});var C=A((et,O)=>{O.exports={version:"1.8.0",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.5.2"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.6.0"},dependencies:{"@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-core":"^3.3.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"d5cc829f7c2177b8db707f04b8fe95246167fdcd"}});import{decompressFrames as N,parseGIF as $}from"gifuct-js";import{HMSPluginUnsupportedTypes as U,HMSVideoPluginType as z}from"@100mslive/hms-video";import"@tensorflow/tfjs-backend-webgl";var _=C(),F=`https://unpkg.com/${_.name}/src`,M="VBProcessor",R="tflite/tflite.js",j="tflite/tflite-simd.js",G="models/selfie_segmentation_landscape.tflite",S=h=>new Promise(function(t,e){let i=document.createElement("script");i.src=h,i.onload=t,i.onerror=e,document.head.appendChild(i)}),W=()=>u(void 0,null,function*(){let h,t=`${F}/${j}`;yield S(t);try{h=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${F}/${R}`,yield S(t),h=yield createTFLiteModule()}return h}),H=()=>u(void 0,null,function*(){let h=`${F}/${G}`,[t,e]=yield Promise.all([W(),fetch(h)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(M,"Input memory offset:",t._getInputMemoryOffset()),console.debug(M,"Input height:",t._getInputHeight()),console.debug(M,"Input width:",t._getInputWidth()),console.debug(M,"Input channels:",t._getInputChannelCount()),t});var y="VBProcessor",q=33,J=C(),Y=214,Q=855,K=120,X=720,Z=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(y,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return u(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(y,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=H(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=U.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return J.name}getPluginType(){return z.TRANSFORM}setBackground(t){return u(this,null,function*(){if(t!=="")if(t==="none")this.log(y,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(y,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>u(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let a=()=>u(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):a()}setImage(t){return u(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>$(e)).then(e=>N(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],a=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,a),o=Math.exp(s-n),r=Math.exp(a-n);this.personMask.data[i*4+3]=255*r/(o+r)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>Y&&this.output.height>K&&this.output.width<Q&&this.output.height<X&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/q?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return u(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,a,n,o;e/i<this.imageAspectRatio?(s=e,a=e/this.imageAspectRatio,n=0,o=(i-a)/2):(a=i,s=i*this.imageAspectRatio,o=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,o,s,a,0,0,this.output.width,this.output.height)}addBlurToBackground(){return u(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),a=Math.floor(s/2),n=t.data,o=t.width,r=t.height,l=o,x=r,T=this.filters.createImageData(l,x),p=T.data,P=i?1:0;for(let d=0;d<x;d=d+1)for(let g=0;g<l;g=g+1){let c=(d*l+g)*4;if(n[c+3]!==0&&g<l&&d<x){let D=d,B=g,L=0,E=0,V=0,w=0;for(let m=0;m<s;m++)for(let f=0;f<s;f++){let v=D+m-a,I=B+f-a;if(v>=0&&v<r&&I>=0&&I<o){let k=(v*o+I)*4,b=e[m*s+f];L+=n[k]*b,E+=n[k+1]*b,V+=n[k+2]*b,w+=n[k+3]*b}}p[c]=L,p[c+1]=E,p[c+2]=V,p[c+3]=w+P*(255-w)}}return T}}};export{Z as HMSVirtualBackgroundPlugin};
+var A=(h,t)=>()=>(t||h((t={exports:{}}).exports,t),t.exports);var u=(h,t,e)=>new Promise((i,s)=>{var a=r=>{try{o(e.next(r))}catch(l){s(l)}},n=r=>{try{o(e.throw(r))}catch(l){s(l)}},o=r=>r.done?i(r.value):Promise.resolve(r.value).then(a,n);o((e=e.apply(h,t)).next())});var C=A((et,O)=>{O.exports={version:"1.8.1",license:"MIT",main:"dist/index.cjs.js",typings:"dist/index.d.ts",files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video":"^0.5.2"},name:"@100mslive/hms-virtual-background",author:"ashish17",module:"dist/index.js",devDependencies:{"@100mslive/hms-video":"0.6.1"},dependencies:{"@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-core":"^3.3.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"b4d98c849c3e4ae4e583b2e2c8f9813193cf1d7c"}});import{decompressFrames as N,parseGIF as $}from"gifuct-js";import{HMSPluginUnsupportedTypes as U,HMSVideoPluginType as z}from"@100mslive/hms-video";import"@tensorflow/tfjs-backend-webgl";var _=C(),F=`https://unpkg.com/${_.name}/src`,M="VBProcessor",R="tflite/tflite.js",j="tflite/tflite-simd.js",G="models/selfie_segmentation_landscape.tflite",S=h=>new Promise(function(t,e){let i=document.createElement("script");i.src=h,i.onload=t,i.onerror=e,document.head.appendChild(i)}),W=()=>u(void 0,null,function*(){let h,t=`${F}/${j}`;yield S(t);try{h=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${F}/${R}`,yield S(t),h=yield createTFLiteModule()}return h}),H=()=>u(void 0,null,function*(){let h=`${F}/${G}`,[t,e]=yield Promise.all([W(),fetch(h)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(M,"Input memory offset:",t._getInputMemoryOffset()),console.debug(M,"Input height:",t._getInputHeight()),console.debug(M,"Input width:",t._getInputWidth()),console.debug(M,"Input channels:",t._getInputChannelCount()),t});var y="VBProcessor",q=33,J=C(),Y=214,Q=855,K=120,X=720,Z=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(y,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return u(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(y,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=H(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=U.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return J.name}getPluginType(){return z.TRANSFORM}setBackground(t){return u(this,null,function*(){if(t!=="")if(t==="none")this.log(y,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(y,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>u(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let a=()=>u(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):a()}setImage(t){return u(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>$(e)).then(e=>N(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],a=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,a),o=Math.exp(s-n),r=Math.exp(a-n);this.personMask.data[i*4+3]=255*r/(o+r)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>Y&&this.output.height>K&&this.output.width<Q&&this.output.height<X&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/q?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return u(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,a,n,o;e/i<this.imageAspectRatio?(s=e,a=e/this.imageAspectRatio,n=0,o=(i-a)/2):(a=i,s=i*this.imageAspectRatio,o=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,o,s,a,0,0,this.output.width,this.output.height)}addBlurToBackground(){return u(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),a=Math.floor(s/2),n=t.data,o=t.width,r=t.height,l=o,x=r,T=this.filters.createImageData(l,x),p=T.data,P=i?1:0;for(let d=0;d<x;d=d+1)for(let g=0;g<l;g=g+1){let c=(d*l+g)*4;if(n[c+3]!==0&&g<l&&d<x){let D=d,B=g,L=0,E=0,V=0,w=0;for(let m=0;m<s;m++)for(let f=0;f<s;f++){let v=D+m-a,I=B+f-a;if(v>=0&&v<r&&I>=0&&I<o){let k=(v*o+I)*4,b=e[m*s+f];L+=n[k]*b,E+=n[k+1]*b,V+=n[k+2]*b,w+=n[k+3]*b}}p[c]=L,p[c+1]=E,p[c+2]=V,p[c+3]=w+P*(255-w)}}return T}}};export{Z as HMSVirtualBackgroundPlugin};
 //# sourceMappingURL=index.js.map

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.8.0",
+  "version": "1.8.1",
   "license": "MIT",
   "main": "dist/index.cjs.js",
   "typings": "dist/index.d.ts",
@@ -30,7 +30,7 @@
   "author": "ashish17",
   "module": "dist/index.js",
   "devDependencies": {
-    "@100mslive/hms-video": "0.6.0"
+    "@100mslive/hms-video": "0.6.1"
   },
   "dependencies": {
     "@tensorflow/tfjs-backend-webgl": "^3.3.0",
@@ -48,5 +48,5 @@
     "defineTFLite.ts",
     "importing.test.ts"
   ],
-  "gitHead": "d5cc829f7c2177b8db707f04b8fe95246167fdcd"
+  "gitHead": "b4d98c849c3e4ae4e583b2e2c8f9813193cf1d7c"
 }