@100mslive/hms-virtual-background 1.13.27 → 1.13.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/cjs/index.js +1 -1
  2. package/dist/cjs/index.js.map +1 -1
  3. package/dist/esm/index.js +1 -1
  4. package/dist/esm/index.js.map +1 -1
  5. package/package.json +4 -4
package/dist/cjs/index.js CHANGED
@@ -1,2 +1,2 @@
1
- "use strict";var P=Object.defineProperty;var J=Object.getOwnPropertyDescriptor;var K=Object.getOwnPropertyNames;var Q=Object.prototype.hasOwnProperty;var Y=(a,t)=>()=>(t||a((t={exports:{}}).exports,t),t.exports),X=(a,t)=>{for(var e in t)P(a,e,{get:t[e],enumerable:!0})},Z=(a,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of K(t))!Q.call(a,s)&&s!==e&&P(a,s,{get:()=>t[s],enumerable:!(i=J(t,s))||i.enumerable});return a};var tt=a=>Z(P({},"__esModule",{value:!0}),a);var o=(a,t,e)=>new Promise((i,s)=>{var r=h=>{try{u(e.next(h))}catch(d){s(d)}},n=h=>{try{u(e.throw(h))}catch(d){s(d)}},u=h=>h.done?i(h.value):Promise.resolve(h.value).then(r,n);u((e=e.apply(a,t)).next())});var L=Y((mt,et)=>{et.exports={version:"1.13.27",license:"MIT",name:"@100mslive/hms-virtual-background",author:"100ms",module:"dist/esm/index.js",main:"dist/cjs/index.js",typings:"dist/index.d.ts",typesVersions:{"*":{hmsvbplugin:["./dist/HMSVBPlugin.d.ts"],hmseffectsplugin:["./dist/HMSEffectsPlugin.d.ts"]}},exports:{".":{import:"./dist/esm/index.js",require:"./dist/cjs/index.js",default:"./dist/esm/index.js",types:"./dist/index.d.ts"},"./hmsvbplugin":{import:"./dist/esm/HMSVBPlugin.js",require:"./dist/cjs/HMSVBPlugin.js",default:"./dist/esm/HMSVBPlugin.js",types:"./dist/HMSVBPlugin.d.ts"},"./hmseffectsplugin":{import:"./dist/esm/HMSEffectsPlugin.js",default:"./dist/esm/HMSEffectsPlugin.js",require:"./dist/cjs/HMSEffectsPlugin.js",types:"./dist/HMSEffectsPlugin.d.ts"}},repository:{type:"git",url:"https://github.com/100mslive/web-sdks.git",directory:"packages/hms-virtual-background"},files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video-store":"0.12.27"},devDependencies:{"@100mslive/hms-video-store":"0.12.27"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","effects-sdk":"3.4.3","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"bc3bda8727cd8240b6cf3cc6d5ae4e511fdc679b"}});var pt={};X(pt,{HMSEffectsPlugin:()=>T,HMSVBPlugin:()=>I,HMSVirtualBackgroundPlugin:()=>D,HMSVirtualBackgroundTypes:()=>p});module.exports=tt(pt);var y=require("gifuct-js"),x=require("@100mslive/hms-video-store"),Ct=require("@tensorflow/tfjs-backend-webgl");var it=L(),A=`https://unpkg.com/${it.name}/src`,C="VBProcessor",st="tflite/tflite.js",at="tflite/tflite-simd.js",nt="models/selfie_segmentation_landscape.tflite",_=a=>new Promise(function(t,e){let i=document.createElement("script");i.src=a,i.onload=t,i.onerror=e,document.head.appendChild(i)}),ot=()=>o(void 0,null,function*(){let a,t=`${A}/${at}`;yield _(t);try{a=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${A}/${st}`,yield _(t),a=yield createTFLiteModule()}return a}),j=()=>o(void 0,null,function*(){let a=`${A}/${nt}`,[t,e]=yield Promise.all([ot(),fetch(a)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var M="VBProcessor",rt=33,ut=L(),ht=214,dt=855,gt=120,lt=720,D=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(M,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return o(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(M,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=j(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=x.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return ut.name}getPluginType(){return x.HMSVideoPluginType.TRANSFORM}setBackground(t){return o(this,null,function*(){if(t!=="")if(t==="none")this.log(M,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(M,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>o(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>o(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,y.parseGIF)(e)).then(e=>(0,y.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,r),u=Math.exp(s-n),h=Math.exp(r-n);this.personMask.data[i*4+3]=255*h/(u+h)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>ht&&this.output.height>gt&&this.output.width<dt&&this.output.height<lt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/rt?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return o(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,n,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,n=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return o(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),n=t.data,u=t.width,h=t.height,d=u,F=h,R=this.filters.createImageData(d,F),m=R.data,$=i?1:0;for(let g=0;g<F;g=g+1)for(let l=0;l<d;l=l+1){let c=(g*d+l)*4;if(n[c+3]!==0&&l<d&&g<F){let q=g,z=l,O=0,N=0,G=0,H=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let B=q+f-r,V=z+k-r;if(B>=0&&B<h&&V>=0&&V<u){let b=(B*u+V)*4,v=e[f*s+k];O+=n[b]*v,N+=n[b+1]*v,G+=n[b+2]*v,H+=n[b+3]*v}}m[c]=O,m[c+1]=N,m[c+2]=G,m[c+3]=H+$*(255-H)}}return R}}};var U=require("@mediapipe/selfie_segmentation"),w=require("gifuct-js"),S=require("@100mslive/hms-video-store");var p=(n=>(n.BLUR="blur",n.NONE="none",n.GIF="gif",n.IMAGE="image",n.VIDEO="video",n.CANVAS="canvas",n))(p||{});var I=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.background="none";this.backgroundType="none";this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case"image":case"canvas":case"video":this.renderBackground(t,this.background);break;case"gif":this.renderGIF(t);break;case"blur":this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType==="none"||this.backgroundType==="blur")return;this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-out",this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}isBlurSupported(){return"filter"in CanvasRenderingContext2D.prototype}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge","Safari"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=S.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return S.HMSVideoPluginType.TRANSFORM}init(){return o(this,null,function*(){this.segmentation||(this.segmentation=new U.SelfieSegmentation({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return o(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case"none":case"blur":this.background=t,this.backgroundType=e;break;case"image":this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType="image";break;case"video":this.log("setting background to video",t),this.backgroundType="none",this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.playsInline=!0,this.background.oncanplaythrough=()=>o(this,null,function*(){if(this.background&&this.background instanceof HTMLVideoElement)try{yield this.background.play(),this.backgroundType="video"}catch(s){this.log("failed to play background",t)}});break;case"canvas":this.background=t,this.backgroundType="canvas";break;case"gif":if(this.log("setting gif to background",t),this.backgroundType="none",this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif";else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${p}`)}})}getBackground(){return this.background}stop(){var t;this.backgroundType!=="blur"&&this.background!=="none"&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background="none",this.backgroundType="none"}processVideoFrame(t,e,i){return o(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType==="none"){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,w.parseGIF)(e)).then(e=>(0,w.decompressFrames)(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!=="blur"||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!=="gif")){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};var W=require("effects-sdk");var E="https://assets.100ms.live/effectsdk/3.4.3/";var T=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new W.tsvb(t),this.onInit=e,this.effects.config({sdk_url:E,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${E}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${E}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return o(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};
1
+ "use strict";var P=Object.defineProperty;var J=Object.getOwnPropertyDescriptor;var K=Object.getOwnPropertyNames;var Q=Object.prototype.hasOwnProperty;var Y=(a,t)=>()=>(t||a((t={exports:{}}).exports,t),t.exports),X=(a,t)=>{for(var e in t)P(a,e,{get:t[e],enumerable:!0})},Z=(a,t,e,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let s of K(t))!Q.call(a,s)&&s!==e&&P(a,s,{get:()=>t[s],enumerable:!(i=J(t,s))||i.enumerable});return a};var tt=a=>Z(P({},"__esModule",{value:!0}),a);var o=(a,t,e)=>new Promise((i,s)=>{var r=h=>{try{u(e.next(h))}catch(d){s(d)}},n=h=>{try{u(e.throw(h))}catch(d){s(d)}},u=h=>h.done?i(h.value):Promise.resolve(h.value).then(r,n);u((e=e.apply(a,t)).next())});var L=Y((mt,et)=>{et.exports={version:"1.13.28",license:"MIT",name:"@100mslive/hms-virtual-background",author:"100ms",module:"dist/esm/index.js",main:"dist/cjs/index.js",typings:"dist/index.d.ts",typesVersions:{"*":{hmsvbplugin:["./dist/HMSVBPlugin.d.ts"],hmseffectsplugin:["./dist/HMSEffectsPlugin.d.ts"]}},exports:{".":{import:"./dist/esm/index.js",require:"./dist/cjs/index.js",default:"./dist/esm/index.js",types:"./dist/index.d.ts"},"./hmsvbplugin":{import:"./dist/esm/HMSVBPlugin.js",require:"./dist/cjs/HMSVBPlugin.js",default:"./dist/esm/HMSVBPlugin.js",types:"./dist/HMSVBPlugin.d.ts"},"./hmseffectsplugin":{import:"./dist/esm/HMSEffectsPlugin.js",default:"./dist/esm/HMSEffectsPlugin.js",require:"./dist/cjs/HMSEffectsPlugin.js",types:"./dist/HMSEffectsPlugin.d.ts"}},repository:{type:"git",url:"https://github.com/100mslive/web-sdks.git",directory:"packages/hms-virtual-background"},files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video-store":"0.12.28"},devDependencies:{"@100mslive/hms-video-store":"0.12.28"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","effects-sdk":"3.4.3","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"c4d7367340117e66de386fc7aaa9b5e95abd19ed"}});var pt={};X(pt,{HMSEffectsPlugin:()=>T,HMSVBPlugin:()=>I,HMSVirtualBackgroundPlugin:()=>D,HMSVirtualBackgroundTypes:()=>p});module.exports=tt(pt);var y=require("gifuct-js"),x=require("@100mslive/hms-video-store"),Ct=require("@tensorflow/tfjs-backend-webgl");var it=L(),A=`https://unpkg.com/${it.name}/src`,C="VBProcessor",st="tflite/tflite.js",at="tflite/tflite-simd.js",nt="models/selfie_segmentation_landscape.tflite",_=a=>new Promise(function(t,e){let i=document.createElement("script");i.src=a,i.onload=t,i.onerror=e,document.head.appendChild(i)}),ot=()=>o(void 0,null,function*(){let a,t=`${A}/${at}`;yield _(t);try{a=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${A}/${st}`,yield _(t),a=yield createTFLiteModule()}return a}),j=()=>o(void 0,null,function*(){let a=`${A}/${nt}`,[t,e]=yield Promise.all([ot(),fetch(a)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var M="VBProcessor",rt=33,ut=L(),ht=214,dt=855,gt=120,lt=720,D=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(M,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return o(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(M,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=j(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=x.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return ut.name}getPluginType(){return x.HMSVideoPluginType.TRANSFORM}setBackground(t){return o(this,null,function*(){if(t!=="")if(t==="none")this.log(M,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(M,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>o(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>o(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,y.parseGIF)(e)).then(e=>(0,y.decompressFrames)(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],n=Math.max(s,r),u=Math.exp(s-n),h=Math.exp(r-n);this.personMask.data[i*4+3]=255*h/(u+h)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>ht&&this.output.height>gt&&this.output.width<dt&&this.output.height<lt&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/rt?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return o(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,n,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,n=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,n=(e-s)/2),this.outputCtx.drawImage(t,n,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return o(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),n=t.data,u=t.width,h=t.height,d=u,F=h,R=this.filters.createImageData(d,F),m=R.data,$=i?1:0;for(let g=0;g<F;g=g+1)for(let l=0;l<d;l=l+1){let c=(g*d+l)*4;if(n[c+3]!==0&&l<d&&g<F){let q=g,z=l,O=0,N=0,G=0,H=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let B=q+f-r,V=z+k-r;if(B>=0&&B<h&&V>=0&&V<u){let b=(B*u+V)*4,v=e[f*s+k];O+=n[b]*v,N+=n[b+1]*v,G+=n[b+2]*v,H+=n[b+3]*v}}m[c]=O,m[c+1]=N,m[c+2]=G,m[c+3]=H+$*(255-H)}}return R}}};var U=require("@mediapipe/selfie_segmentation"),w=require("gifuct-js"),S=require("@100mslive/hms-video-store");var p=(n=>(n.BLUR="blur",n.NONE="none",n.GIF="gif",n.IMAGE="image",n.VIDEO="video",n.CANVAS="canvas",n))(p||{});var I=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.background="none";this.backgroundType="none";this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case"image":case"canvas":case"video":this.renderBackground(t,this.background);break;case"gif":this.renderGIF(t);break;case"blur":this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType==="none"||this.backgroundType==="blur")return;this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-out",this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}isBlurSupported(){return"filter"in CanvasRenderingContext2D.prototype}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge","Safari"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=S.HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return S.HMSVideoPluginType.TRANSFORM}init(){return o(this,null,function*(){this.segmentation||(this.segmentation=new U.SelfieSegmentation({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return o(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case"none":case"blur":this.background=t,this.backgroundType=e;break;case"image":this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType="image";break;case"video":this.log("setting background to video",t),this.backgroundType="none",this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.playsInline=!0,this.background.oncanplaythrough=()=>o(this,null,function*(){if(this.background&&this.background instanceof HTMLVideoElement)try{yield this.background.play(),this.backgroundType="video"}catch(s){this.log("failed to play background",t)}});break;case"canvas":this.background=t,this.backgroundType="canvas";break;case"gif":if(this.log("setting gif to background",t),this.backgroundType="none",this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif";else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${p}`)}})}getBackground(){return this.background}stop(){var t;this.backgroundType!=="blur"&&this.background!=="none"&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background="none",this.backgroundType="none"}processVideoFrame(t,e,i){return o(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType==="none"){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return o(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>(0,w.parseGIF)(e)).then(e=>(0,w.decompressFrames)(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!=="blur"||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!=="gif")){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};var W=require("effects-sdk");var E="https://assets.100ms.live/effectsdk/3.4.3/";var T=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new W.tsvb(t),this.onInit=e,this.effects.config({sdk_url:E,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${E}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${E}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return o(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};
2
2
  //# sourceMappingURL=index.js.map
package/dist/cjs/index.js.map CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../package.json", "../../src/index.ts", "../../src/HMSVirtualBackgroundPlugin.ts", "../../src/defineTFLite.ts", "../../src/HMSVBPlugin.ts", "../../src/interfaces.ts", "../../src/HMSEffectsPlugin.ts", "../../src/constants.ts"],
4
- "sourcesContent": ["{\n \"version\": \"1.13.27\",\n \"license\": \"MIT\",\n \"name\": \"@100mslive/hms-virtual-background\",\n \"author\": \"100ms\",\n \"module\": \"dist/esm/index.js\",\n \"main\": \"dist/cjs/index.js\",\n \"typings\": \"dist/index.d.ts\",\n \"typesVersions\": {\n \"*\": {\n \"hmsvbplugin\": [\n \"./dist/HMSVBPlugin.d.ts\"\n ],\n \"hmseffectsplugin\": [\n \"./dist/HMSEffectsPlugin.d.ts\"\n ]\n }\n },\n \"exports\": {\n \".\": {\n \"import\": \"./dist/esm/index.js\",\n \"require\": \"./dist/cjs/index.js\",\n \"default\": \"./dist/esm/index.js\",\n \"types\": \"./dist/index.d.ts\"\n },\n \"./hmsvbplugin\": {\n \"import\": \"./dist/esm/HMSVBPlugin.js\",\n \"require\": \"./dist/cjs/HMSVBPlugin.js\",\n \"default\": \"./dist/esm/HMSVBPlugin.js\",\n \"types\": \"./dist/HMSVBPlugin.d.ts\"\n },\n \"./hmseffectsplugin\": {\n \"import\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"default\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"require\": \"./dist/cjs/HMSEffectsPlugin.js\",\n \"types\": \"./dist/HMSEffectsPlugin.d.ts\"\n }\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"https://github.com/100mslive/web-sdks.git\",\n \"directory\": \"packages/hms-virtual-background\"\n },\n \"files\": [\n \"dist\",\n \"src/tflite\",\n \"src/models\"\n ],\n \"scripts\": {\n \"start\": \"concurrently \\\"yarn dev\\\" \\\"yarn types\\\"\",\n \"dev\": \"node ../../scripts/dev\",\n \"build:only\": \"node ../../scripts/build\",\n \"build\": \"yarn build:only && yarn types:build\",\n \"types\": \"tsc -w\",\n \"types:build\": \"tsc -p tsconfig.json\",\n \"test\": \"jest --maxWorkers=1 --passWithNoTests\",\n \"lint\": \"eslint -c ../../.eslintrc .\",\n \"lint:fix\": \"yarn lint --fix\",\n \"prepare\": \"yarn build\",\n \"size\": \"size-limit\",\n \"analyze\": \"size-limit --why\",\n \"format\": \"prettier --write src/**/*.ts\"\n },\n \"peerDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.27\"\n },\n \"devDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.27\"\n },\n \"dependencies\": {\n \"@mediapipe/selfie_segmentation\": \"^0.1.1632777926\",\n \"@tensorflow-models/body-segmentation\": \"^1.0.1\",\n \"@tensorflow/tfjs-backend-webgl\": \"^3.3.0\",\n \"@tensorflow/tfjs-converter\": \"^3.19.0\",\n \"@tensorflow/tfjs-core\": \"^3.19.0\",\n \"@webassemblyjs/helper-wasm-bytecode\": \"1.11.1\",\n \"@webassemblyjs/wasm-gen\": \"1.11.1\",\n \"effects-sdk\": \"3.4.3\",\n \"gifuct-js\": \"^2.1.2\",\n \"wasm-check\": \"^2.0.2\"\n },\n \"eslintIgnore\": [\n \"tflite.js\",\n \"tflite-simd.js\",\n \"tflite.wasm\",\n \"tflite-simd.wasm\",\n \"defineTFLite.ts\",\n \"importing.test.ts\"\n ],\n \"gitHead\": \"bc3bda8727cd8240b6cf3cc6d5ae4e511fdc679b\"\n}\n", "export * from './HMSVirtualBackgroundPlugin';\nexport { HMSVBPlugin } from './HMSVBPlugin';\nexport { HMSVirtualBackgroundTypes } from './interfaces';\nexport type { HMSVirtualBackground } from './interfaces';\nexport { HMSEffectsPlugin } from './HMSEffectsPlugin';\nexport type { HMSEffectsBackground } from './HMSEffectsPlugin';\n", "/* eslint-disable complexity */\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport '@tensorflow/tfjs-backend-webgl';\nimport { loadTFLite } from './defineTFLite';\n\nconst TAG = 'VBProcessor';\nconst DEFAULT_DELAY = 33;\nconst pkg = require('../package.json');\n\nconst minVideoWidthForSharpening = 214;\nconst maxVideoWidthForSharpening = 855;\nconst minVideoHeightForSharpening = 120;\nconst maxVideoHeightForSharpening = 720;\n\nexport class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {\n background: string | HTMLImageElement;\n personMaskWidth: number;\n personMaskHeight: number;\n isVirtualBackground: boolean;\n backgroundImage: HTMLImageElement | null;\n backgroundVideo: HTMLVideoElement | null;\n backgroundType = 'none';\n loadModelCalled: boolean;\n blurValue: any;\n tfLite: any;\n tfLitePromise: any;\n modelName: string;\n\n input: HTMLCanvasElement | null;\n output: HTMLCanvasElement | null;\n outputCtx: CanvasRenderingContext2D | null;\n timerID: number;\n imageAspectRatio: number;\n\n personMaskPixelCount: number;\n personMask: ImageData;\n personMaskCanvas: HTMLCanvasElement;\n personMaskCtx: any;\n filters: any;\n enableSharpening?: boolean | false;\n\n gifFrames: any;\n gifFramesIndex: number;\n gifFrameImageData: any;\n tempGifCanvas: HTMLCanvasElement;\n tempGifContext: any;\n giflocalCount: number;\n\n constructor(background: string, enableSharpening = false) {\n this.background = background;\n this.enableSharpening = enableSharpening;\n\n this.backgroundImage = null;\n this.backgroundVideo = null;\n\n this.personMaskWidth = 256;\n this.personMaskHeight = 144;\n this.isVirtualBackground = false;\n this.blurValue = '10px';\n this.loadModelCalled = false;\n this.tfLite = null;\n this.modelName = 'landscape-segmentation';\n\n this.outputCtx = null;\n this.input = null;\n this.output = null;\n this.timerID = 0;\n this.imageAspectRatio = 1;\n\n this.personMaskPixelCount = this.personMaskWidth * this.personMaskHeight;\n this.personMask = new ImageData(this.personMaskWidth, this.personMaskHeight);\n this.personMaskCanvas = document.createElement('canvas');\n this.personMaskCanvas.width = this.personMaskWidth;\n this.personMaskCanvas.height = this.personMaskHeight;\n this.personMaskCtx = this.personMaskCanvas.getContext('2d');\n\n this.filters = {};\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n this.giflocalCount = 0;\n this.enableSharpening = enableSharpening;\n\n this.log(TAG, 'Virtual Background plugin created');\n this.setBackground(this.background);\n }\n\n async init(): Promise<void> {\n if (!this.loadModelCalled) {\n this.log(TAG, 'PREVIOUS LOADED MODEL IS ', this.tfLite);\n this.loadModelCalled = true;\n this.tfLitePromise = loadTFLite();\n this.tfLite = await this.tfLitePromise;\n } else {\n //either it is loading or loaded\n await this.tfLitePromise;\n }\n if (this.enableSharpening) {\n this.initSharpenFilter();\n }\n }\n\n /*\n @depreceated\n */\n isSupported(): boolean {\n //support chrome, firefox, edge TODO: check this\n return (\n navigator.userAgent.indexOf('Chrome') !== -1 ||\n navigator.userAgent.indexOf('Firefox') !== -1 ||\n navigator.userAgent.indexOf('Edg') !== -1 ||\n navigator.userAgent.indexOf('Edge') !== -1\n );\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return pkg.name;\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async setBackground(path?: string | HTMLImageElement | HTMLVideoElement) {\n if (path !== '') {\n if (path === 'none') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'none';\n this.backgroundType = 'none';\n this.isVirtualBackground = false;\n } else if (path === 'blur') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'blur';\n this.backgroundType = 'blur';\n this.isVirtualBackground = false;\n } else if (path instanceof HTMLImageElement) {\n this.log('setting background to image', path);\n const img = await this.setImage(path as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.isVirtualBackground = true;\n this.backgroundImage = img;\n this.backgroundType = 'image';\n }\n } else if (path instanceof HTMLVideoElement) {\n this.log('setting background to video', path);\n this.backgroundVideo = path as HTMLVideoElement;\n this.backgroundVideo.crossOrigin = 'anonymous';\n this.backgroundVideo.muted = true;\n this.backgroundVideo.loop = true;\n this.backgroundVideo.oncanplaythrough = async () => {\n if (this.backgroundVideo != null) {\n await this.backgroundVideo!.play();\n this.isVirtualBackground = true;\n this.backgroundType = 'video';\n }\n };\n } else {\n console.log('setting gif to background');\n this.gifFrames = await this.setGiF(path as string);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = 'gif';\n this.isVirtualBackground = true;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n\n stop(): void {\n if (this.isVirtualBackground) {\n this.backgroundImage?.removeAttribute('src');\n this.backgroundVideo?.removeAttribute('src');\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.loop = false;\n this.backgroundVideo = null;\n }\n }\n if (this.outputCtx) {\n this.outputCtx!.fillStyle = `rgb(0, 0, 0)`;\n this.outputCtx!.fillRect(0, 0, this.output!.width, this.output!.height);\n }\n\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.giflocalCount = 0;\n this.gifFramesIndex = 0;\n }\n\n processVideoFrame(\n input: HTMLCanvasElement,\n output: HTMLCanvasElement,\n skipProcessing?: boolean,\n ): Promise<void> | void {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n\n this.input = input;\n this.output = output;\n\n const ctx = output.getContext('2d');\n if (ctx!.canvas.width !== input.width) {\n ctx!.canvas.width = input.width;\n }\n if (ctx!.canvas.height !== input.height) {\n ctx!.canvas.height = input.height;\n }\n\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.width = input.width;\n this.backgroundVideo!.height = input.height;\n }\n\n this.outputCtx = ctx!;\n this.imageAspectRatio = input.width / input.height;\n if (this.imageAspectRatio <= 0) {\n throw new Error('Invalid input width/height');\n }\n\n const process = async () => {\n await this.runSegmentation(skipProcessing);\n };\n\n if (this.background === 'none' && !this.isVirtualBackground) {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n this.outputCtx!.drawImage(input, 0, 0, input.width, input.height);\n } else {\n process();\n }\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private setGiF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(tag: string, ...data: any[]) {\n console.info(tag, ...data);\n }\n\n private resizeInputData() {\n this.personMaskCtx!.drawImage(\n this.input,\n 0,\n 0,\n this.input!.width,\n this.input!.height,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n );\n\n const imageData = this.personMaskCtx!.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight);\n //\n const inputMemoryOffset = this.tfLite._getInputMemoryOffset() / 4;\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3] = imageData.data[i * 4] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 1] = imageData.data[i * 4 + 1] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 2] = imageData.data[i * 4 + 2] / 255;\n }\n }\n private infer(skipProcessing?: boolean) {\n if (!skipProcessing) {\n this.tfLite._runInference();\n }\n const outputMemoryOffset = this.tfLite._getOutputMemoryOffset() / 4;\n\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n if (this.modelName === 'meet') {\n const background = this.tfLite.HEAPF32[outputMemoryOffset + i * 2];\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i * 2 + 1];\n const shift = Math.max(background, person);\n const backgroundExp = Math.exp(background - shift);\n const personExp = Math.exp(person - shift);\n // Sets only the alpha component of each pixel.\n this.personMask.data[i * 4 + 3] = (255 * personExp) / (backgroundExp + personExp);\n } else if (this.modelName === 'landscape-segmentation') {\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i];\n this.personMask.data[i * 4 + 3] = 255 * person;\n }\n }\n\n this.personMaskCtx!.putImageData(this.personMask, 0, 0);\n }\n\n private postProcessing() {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n\n if (this.isVirtualBackground) {\n this.outputCtx!.filter = 'blur(4px)';\n } else {\n this.outputCtx!.filter = 'blur(8px)';\n }\n this.drawPersonMask();\n this.outputCtx!.globalCompositeOperation = 'source-in';\n this.outputCtx!.filter = 'none';\n // //Draw the foreground\n this.outputCtx!.drawImage(this.input!, 0, 0);\n\n if (\n this.enableSharpening &&\n this.output!.width > minVideoWidthForSharpening && // minimum and maximum resolution to enable sharpening filter\n this.output!.height > minVideoHeightForSharpening &&\n this.output!.width < maxVideoWidthForSharpening &&\n this.output!.height < maxVideoHeightForSharpening\n ) {\n this.sharpenFilter();\n }\n\n // //Draw the background\n this.drawSegmentedBackground();\n }\n\n private sharpenFilter() {\n // adding sharpening filter to each frame to improve edges and brightness\n // The basic idea is that you take the weighed sum of a rectangle of pixels from the source image and use that as the output value using convolution filter\n // It is applied intermediate output with black background and only mask data in frame\n // Filter currently used is 3 x 3 sharpening filter with values as shown:\n // [ 0, -1, 0,\n // -1, 5, -1,\n // 0, -1, 0 ]\n const outputImageData = this.outputCtx!.getImageData(0, 0, this.output!.width, this.output!.height);\n\n // filters you may try\n // [-1, -1, -1, -1, 9, -1, -1, -1, -1]\n //[0, -1, 0, -1, 5, -1, 0, -1, 0]\n const output = this.filters.convolute(outputImageData);\n\n this.outputCtx!.putImageData(output, 0, 0);\n }\n\n private drawPersonMask() {\n this.outputCtx!.drawImage(\n this.personMaskCanvas,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private drawSegmentedBackground() {\n this.outputCtx!.globalCompositeOperation = 'destination-over';\n this.outputCtx!.imageSmoothingEnabled = true;\n this.outputCtx!.imageSmoothingQuality = 'high';\n if (this.isVirtualBackground) {\n if (this.backgroundType === 'video' && this.backgroundVideo != null && this.backgroundVideo!.readyState >= 4) {\n this.fitVideoToBackground();\n } else if (this.backgroundType === 'image') {\n this.fitImageToBackground();\n } else if (this.backgroundType === 'gif') {\n if (this.giflocalCount > this.gifFrames[this.gifFramesIndex].delay / DEFAULT_DELAY) {\n this.gifFramesIndex++;\n if (this.gifFramesIndex >= this.gifFrames.length) {\n this.gifFramesIndex = 0;\n }\n this.giflocalCount = 0;\n } else {\n this.giflocalCount++;\n }\n this.fitGifToBackground();\n }\n } else {\n this.addBlurToBackground();\n }\n }\n\n private async runSegmentation(skipProcessing?: boolean) {\n if (this.tfLite) {\n // const start = performance.now();\n\n this.resizeInputData();\n await this.infer(skipProcessing);\n this.postProcessing();\n // const end = performance.now();\n // this.log(TAG,\"time taken\",end -start);\n }\n }\n\n private fitVideoToBackground() {\n this.fitData(this.backgroundVideo, this.backgroundVideo!.videoWidth, this.backgroundVideo!.videoHeight);\n }\n\n private fitImageToBackground() {\n this.fitData(this.backgroundImage, this.backgroundImage!.width, this.backgroundImage!.height);\n }\n\n private fitGifToBackground() {\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas!.width = dims.width;\n this.tempGifCanvas!.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n\n this.fitData(this.tempGifCanvas, this.gifFrameImageData!.width, this.gifFrameImageData!.height);\n }\n\n private fitData(data: any, dataWidth: number, dataHeight: number) {\n let inputWidth: any, inputHeight: any, xoffset: any, yoffset: any;\n if (dataWidth / dataHeight < this.imageAspectRatio) {\n inputWidth = dataWidth;\n inputHeight = dataWidth / this.imageAspectRatio;\n xoffset = 0;\n yoffset = (dataHeight - inputHeight) / 2;\n } else {\n inputHeight = dataHeight;\n inputWidth = dataHeight * this.imageAspectRatio;\n yoffset = 0;\n xoffset = (dataWidth - inputWidth) / 2;\n }\n this.outputCtx!.drawImage(\n data,\n xoffset,\n yoffset,\n inputWidth,\n inputHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private async addBlurToBackground() {\n let blurValue = '15px';\n if (this.input!.width <= 160) {\n blurValue = '5px';\n } else if (this.input!.width <= 320) {\n blurValue = '10px';\n } else if (this.input!.width <= 640) {\n blurValue = '15px';\n } else if (this.input!.width <= 960) {\n blurValue = '20px';\n } else if (this.input!.width <= 1280) {\n blurValue = '25px';\n } else if (this.input!.width <= 1920) {\n blurValue = '30px';\n }\n\n this.outputCtx!.filter = `blur(${blurValue})`;\n this.outputCtx!.drawImage(this.input!, 0, 0, this.output!.width, this.output!.height);\n }\n\n private initSharpenFilter(): any {\n this.filters.tmpCanvas = document.createElement('canvas');\n this.filters.tmpCtx = this.filters.tmpCanvas.getContext('2d');\n\n this.filters.createImageData = (w: number, h: number) => {\n return this.filters.tmpCtx.createImageData(w, h);\n };\n\n this.filters.convolute = (pixels: ImageData, weights = [0, -1, 0, -1, 5, -1, 0, -1, 0], opaque: boolean) => {\n const side = Math.round(Math.sqrt(weights.length));\n\n const halfSide = Math.floor(side / 2);\n const src = pixels.data;\n const sw = pixels.width;\n const sh = pixels.height;\n // pad output by the convolution matrix\n const w = sw;\n const h = sh;\n const output = this.filters.createImageData(w, h);\n const dst = output.data;\n // go through the destination image pixels\n const alphaFac = opaque ? 1 : 0;\n for (let y = 0; y < h; y = y + 1) {\n for (let x = 0; x < w; x = x + 1) {\n const dstOff = (y * w + x) * 4;\n\n if (src[dstOff + 3] === 0) {\n continue;\n } else if (x < w && y < h) {\n const sy = y;\n const sx = x;\n\n // calculate the weighed sum of the source image pixels that\n // fall under the convolution matrix\n let r = 0,\n g = 0,\n b = 0,\n a = 0;\n for (let cy = 0; cy < side; cy++) {\n for (let cx = 0; cx < side; cx++) {\n const scy = sy + cy - halfSide;\n const scx = sx + cx - halfSide;\n if (scy >= 0 && scy < sh && scx >= 0 && scx < sw) {\n const srcOff = (scy * sw + scx) * 4;\n const wt = weights[cy * side + cx];\n r += src[srcOff] * wt;\n g += src[srcOff + 1] * wt;\n b += src[srcOff + 2] * wt;\n a += src[srcOff + 3] * wt;\n }\n }\n }\n dst[dstOff] = r;\n dst[dstOff + 1] = g;\n dst[dstOff + 2] = b;\n dst[dstOff + 3] = a + alphaFac * (255 - a);\n }\n }\n }\n return output;\n };\n }\n}\n", "const pkg = require('../package.json');\nconst BASE_URL = `https://unpkg.com/${pkg.name}/src`;\nconst TAG = 'VBProcessor';\nconst TFLITE_JS_FILE = 'tflite/tflite.js';\nconst TFLITE_SIMD_JS_FILE = 'tflite/tflite-simd.js';\nconst MODEL_FILE_NAME = 'models/selfie_segmentation_landscape.tflite';\n\nconst loadScript = (src: string) => {\n return new Promise(function (resolve, reject) {\n const s = document.createElement('script');\n s.src = src;\n s.onload = resolve;\n s.onerror = reject;\n document.head.appendChild(s);\n });\n};\n\nconst loadTFLiteModel = async () => {\n let tfLite: any;\n let path = `${BASE_URL}/${TFLITE_SIMD_JS_FILE}`;\n await loadScript(path);\n try {\n //@ts-ignore\n tfLite = await createTFLiteSIMDModule();\n } catch {\n console.warn('SIMD not supported. You may experience poor virtual background effect.');\n path = `${BASE_URL}/${TFLITE_JS_FILE}`;\n await loadScript(path);\n // @ts-ignore\n tfLite = await createTFLiteModule();\n }\n return tfLite;\n};\n\nconst loadTFLite = async () => {\n const modelPath = `${BASE_URL}/${MODEL_FILE_NAME}`;\n const [tfLite, modelResponse] = await Promise.all([loadTFLiteModel(), fetch(modelPath)]);\n\n const model = await modelResponse.arrayBuffer();\n const modelBufferOffset = tfLite._getModelBufferMemoryOffset();\n tfLite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);\n tfLite._loadModel(model.byteLength);\n\n console.debug(TAG, 'Input memory offset:', tfLite._getInputMemoryOffset());\n console.debug(TAG, 'Input height:', tfLite._getInputHeight());\n console.debug(TAG, 'Input width:', tfLite._getInputWidth());\n console.debug(TAG, 'Input channels:', tfLite._getInputChannelCount());\n\n return tfLite;\n};\n\nexport { loadTFLite };\n", "/* eslint-disable complexity */\nimport { Results as MediaPipeResults, SelfieSegmentation } from '@mediapipe/selfie_segmentation';\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport { HMSBackgroundInput, HMSVirtualBackground, HMSVirtualBackgroundTypes } from './interfaces';\n\nexport class HMSVBPlugin implements HMSVideoPlugin {\n private TAG = '[HMSVBPlugin]';\n private background: HMSVirtualBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType: HMSVirtualBackgroundTypes = HMSVirtualBackgroundTypes.NONE;\n private segmentation!: SelfieSegmentation;\n private outputCanvas?: HTMLCanvasElement;\n private outputCtx?: CanvasRenderingContext2D | null;\n\n private gifFrames: any;\n private gifFramesIndex: number;\n private gifFrameImageData: any;\n private tempGifCanvas: HTMLCanvasElement;\n private tempGifContext: CanvasRenderingContext2D | null;\n private prevResults?: MediaPipeResults;\n private input?: HTMLCanvasElement;\n\n constructor(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n this.background = background;\n this.backgroundType = backgroundType;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n\n this.setBackground(this.background, this.backgroundType);\n this.log('Virtual background plugin initialised');\n }\n\n isSupported(): boolean {\n return this.checkSupport().isSupported;\n }\n\n isBlurSupported(): boolean {\n return 'filter' in CanvasRenderingContext2D.prototype;\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge', 'Safari'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return 'HMSVB';\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async init(): Promise<void> {\n if (!this.segmentation) {\n this.segmentation = new SelfieSegmentation({\n locateFile: (file: string) => {\n return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${file}`;\n },\n });\n this.segmentation.setOptions({ selfieMode: false, modelSelection: 1 });\n this.segmentation.onResults(this.handleResults);\n }\n }\n\n /**\n * For bgType HMSVirtualBackgroundTypes.IMAGE pass bg as an image element\n * For bgType HMSVirtualBackgroundTypes.VIDEO pass video\n * For bgType HMSVirtualBackgroundTypes.GIF pass the gif url\n * @param {HMSVirtualBackground} background\n * @param {HMSVirtualBackgroundTypes} backgroundType\n */\n async setBackground(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n if (!background) {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n this.prevResults = undefined;\n switch (backgroundType) {\n case HMSVirtualBackgroundTypes.NONE:\n case HMSVirtualBackgroundTypes.BLUR:\n this.background = background;\n this.backgroundType = backgroundType;\n break;\n case HMSVirtualBackgroundTypes.IMAGE:\n this.log('setting background to image', background);\n // eslint-disable-next-line no-case-declarations\n const img = await this.setImage(background as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.background = img;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n }\n break;\n case HMSVirtualBackgroundTypes.VIDEO:\n this.log('setting background to video', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as HTMLVideoElement;\n this.background.crossOrigin = 'anonymous';\n this.background.muted = true;\n this.background.loop = true;\n this.background.playsInline = true;\n this.background.oncanplaythrough = async () => {\n if (this.background && this.background instanceof HTMLVideoElement) {\n try {\n await this.background.play();\n this.backgroundType = HMSVirtualBackgroundTypes.VIDEO;\n } catch (e) {\n this.log('failed to play background', background);\n }\n }\n };\n break;\n case HMSVirtualBackgroundTypes.CANVAS:\n this.background = background;\n this.backgroundType = HMSVirtualBackgroundTypes.CANVAS;\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.log('setting gif to background', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as string;\n this.gifFrames = await this.loadGIF(this.background);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = HMSVirtualBackgroundTypes.GIF;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n break;\n default:\n this.log(\n `backgroundType did not match with any of the supported background types - ${HMSVirtualBackgroundTypes}`,\n );\n }\n }\n\n getBackground() {\n return this.background;\n }\n\n stop(): void {\n if (this.backgroundType !== HMSVirtualBackgroundTypes.BLUR && this.background !== HMSVirtualBackgroundTypes.NONE) {\n this.segmentation?.reset();\n }\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.background = HMSVirtualBackgroundTypes.NONE;\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n }\n\n async processVideoFrame(input: HTMLCanvasElement, output: HTMLCanvasElement, skipProcessing?: boolean) {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n this.input = input;\n output.width = input.width;\n output.height = input.height;\n this.outputCanvas = output;\n this.outputCtx = output.getContext('2d');\n if (skipProcessing && this.prevResults) {\n this.handleResults(this.prevResults);\n return;\n }\n if (this.backgroundType === HMSVirtualBackgroundTypes.NONE) {\n this.outputCtx?.drawImage(input, 0, 0, input.width, input.height);\n return;\n }\n await this.segmentation.send({ image: input });\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private handleResults = (results: MediaPipeResults) => {\n if (!this.outputCanvas || !this.outputCtx) {\n return;\n }\n this.outputCtx.save();\n this.outputCtx.clearRect(0, 0, this.outputCanvas.width, this.outputCanvas.height);\n switch (this.backgroundType) {\n case HMSVirtualBackgroundTypes.IMAGE:\n case HMSVirtualBackgroundTypes.CANVAS:\n case HMSVirtualBackgroundTypes.VIDEO:\n this.renderBackground(results, this.background as HMSBackgroundInput);\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.renderGIF(results);\n break;\n case HMSVirtualBackgroundTypes.BLUR:\n this.renderBlur(results);\n break;\n }\n this.outputCtx.restore();\n this.prevResults = results;\n };\n\n private loadGIF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(...data: any[]) {\n console.debug(this.TAG, ...data);\n }\n\n private renderBackground = (results: MediaPipeResults, background: HMSBackgroundInput) => {\n if (\n !this.input ||\n !this.outputCanvas ||\n !this.outputCtx ||\n this.backgroundType === HMSVirtualBackgroundTypes.NONE ||\n this.backgroundType === HMSVirtualBackgroundTypes.BLUR\n ) {\n return;\n }\n this.outputCtx.filter = 'none';\n this.outputCtx.imageSmoothingEnabled = true;\n this.outputCtx.imageSmoothingQuality = 'high';\n // Only overwrite existing pixels.\n this.outputCtx.globalCompositeOperation = 'source-out';\n const bgWidth = background instanceof HTMLVideoElement ? background.videoWidth : background.width;\n const bgHeight = background instanceof HTMLVideoElement ? background.videoHeight : background.height;\n\n this.outputCtx.drawImage(\n background,\n 0,\n 0,\n bgWidth,\n bgHeight,\n 0,\n 0,\n this.outputCanvas.width,\n this.outputCanvas.height,\n );\n this.outputCtx.globalCompositeOperation = 'destination-out';\n this.outputCtx.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n // Only overwrite missing pixels.\n this.outputCtx.globalCompositeOperation = 'destination-atop';\n this.outputCtx.drawImage(this.input, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n };\n\n private renderBlur(results: MediaPipeResults) {\n if (!this.outputCanvas || !this.outputCtx || this.backgroundType !== HMSVirtualBackgroundTypes.BLUR) {\n return;\n }\n this.outputCtx!.filter = 'none';\n this.outputCtx!.globalCompositeOperation = 'source-out';\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.globalCompositeOperation = 'destination-atop';\n this.outputCtx?.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.filter = `blur(${Math.floor(this.outputCanvas.width / 160) * 5}px)`;\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n }\n\n private renderGIF(results: MediaPipeResults) {\n if (\n !this.outputCanvas ||\n !this.outputCtx ||\n !this.tempGifContext ||\n this.backgroundType !== HMSVirtualBackgroundTypes.GIF\n ) {\n return;\n }\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas.width = dims.width;\n this.tempGifCanvas.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n this.gifFramesIndex = (this.gifFramesIndex + 1) % this.gifFrames.length;\n this.renderBackground(results, this.tempGifCanvas);\n }\n}\n\nexport { HMSVirtualBackgroundTypes };\n", "export type HMSBackgroundInput = HTMLImageElement | HTMLVideoElement | HTMLCanvasElement;\nexport enum HMSVirtualBackgroundTypes {\n BLUR = 'blur',\n NONE = 'none',\n GIF = 'gif',\n IMAGE = 'image',\n VIDEO = 'video',\n CANVAS = 'canvas',\n}\n\nexport type HMSVirtualBackground =\n | string\n | HMSVirtualBackgroundTypes.BLUR\n | HMSVirtualBackgroundTypes.NONE\n | HMSBackgroundInput;\n", "import { tsvb } from 'effects-sdk';\nimport { HMSMediaStreamPlugin } from '@100mslive/hms-video-store';\nimport { EFFECTS_SDK_ASSETS } from './constants';\nimport { HMSVirtualBackgroundTypes } from './interfaces';\n\nexport type HMSEffectsBackground = string | MediaStream | MediaStreamTrack | HTMLVideoElement;\n\nexport class HMSEffectsPlugin implements HMSMediaStreamPlugin {\n private effects: tsvb;\n // Ranges from 0 to 1, inclusive\n private blurAmount = 0;\n private background: HMSEffectsBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType = HMSVirtualBackgroundTypes.NONE;\n private preset: 'balanced' | 'quality' = 'balanced';\n private initialised = false;\n private intervalId: NodeJS.Timer | null = null;\n private onInit;\n private onResolutionChangeCallback?: (width: number, height: number) => void;\n private canvas: HTMLCanvasElement;\n\n constructor(effectsSDKKey: string, onInit?: () => void) {\n this.effects = new tsvb(effectsSDKKey);\n this.onInit = onInit;\n this.effects.config({\n sdk_url: EFFECTS_SDK_ASSETS,\n models: {\n colorcorrector: '',\n facedetector: '',\n lowlighter: '',\n },\n wasmPaths: {\n 'ort-wasm.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm.wasm`,\n 'ort-wasm-simd.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd.wasm`,\n },\n });\n this.canvas = document.createElement('canvas');\n this.effects.onError(err => {\n // currently logging info type messages as well\n if (!err.type || err.type === 'error') {\n console.error('[HMSEffectsPlugin]', err);\n }\n });\n this.effects.cache();\n this.effects.onReady = () => {\n if (this.effects) {\n this.initialised = true;\n this.onInit?.();\n this.effects.run();\n this.effects.setBackgroundFitMode('fill');\n this.effects.setSegmentationPreset(this.preset);\n this.applyEffect();\n }\n };\n }\n\n getName(): string {\n return 'HMSEffects';\n }\n\n private executeAfterInit(callback: () => void) {\n if (this.initialised) {\n callback();\n }\n\n if (this.intervalId !== null) {\n clearInterval(this.intervalId);\n }\n this.intervalId = setInterval(() => {\n if (this.initialised) {\n clearInterval(this.intervalId!);\n callback();\n }\n }, 100);\n }\n\n removeBlur() {\n this.blurAmount = 0;\n this.executeAfterInit(() => {\n this.effects.clearBlur();\n });\n }\n\n removeBackground() {\n this.background = '';\n this.executeAfterInit(() => {\n this.effects.clearBackground();\n });\n }\n\n /**\n * @param blur ranges between 0 and 1\n */\n setBlur(blur: number) {\n this.blurAmount = blur;\n this.backgroundType = HMSVirtualBackgroundTypes.BLUR;\n this.removeBackground();\n this.executeAfterInit(() => {\n this.effects.setBlur(this.blurAmount);\n });\n }\n\n /**\n * @param preset can be 'quality' or 'balanced'. The 'quality' preset has better quality but higher CPU usage than 'balanced'\n */\n async setPreset(preset: 'quality' | 'balanced') {\n this.preset = preset;\n return new Promise((resolve, reject) => {\n this.executeAfterInit(() => {\n this.effects.setSegmentationPreset(this.preset).then(resolve).catch(reject);\n });\n });\n }\n\n onResolutionChange(callback: (width: number, height: number) => void) {\n this.onResolutionChangeCallback = callback;\n }\n\n getPreset() {\n return this.preset;\n }\n\n removeEffects() {\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.removeBackground();\n this.removeBlur();\n }\n\n setBackground(url: HMSEffectsBackground) {\n this.background = url;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n this.removeBlur();\n this.executeAfterInit(() => {\n this.effects.setBackground(this.background);\n });\n }\n\n getBlurAmount() {\n return this.blurAmount;\n }\n\n getBackground() {\n return this.background || this.backgroundType;\n }\n\n private updateCanvas(stream: MediaStream) {\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.canvas.width = width!;\n this.canvas.height = height!;\n this.effects.useStream(stream);\n this.effects.toCanvas(this.canvas);\n }\n\n apply(stream: MediaStream): MediaStream {\n this.effects.clear();\n this.applyEffect();\n this.effects.onChangeInputResolution(() => {\n this.updateCanvas(stream);\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.onResolutionChangeCallback?.(width!, height!);\n });\n this.updateCanvas(stream);\n return this.canvas.captureStream(30) || stream;\n }\n\n stop() {\n this.removeEffects();\n this.executeAfterInit(() => {\n this.effects.stop();\n });\n }\n\n private applyEffect() {\n if (this.blurAmount) {\n this.setBlur(this.blurAmount);\n } else if (this.background) {\n this.setBackground(this.background);\n }\n }\n}\n", "export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/3.4.3/';\n"],
4
+ "sourcesContent": ["{\n \"version\": \"1.13.28\",\n \"license\": \"MIT\",\n \"name\": \"@100mslive/hms-virtual-background\",\n \"author\": \"100ms\",\n \"module\": \"dist/esm/index.js\",\n \"main\": \"dist/cjs/index.js\",\n \"typings\": \"dist/index.d.ts\",\n \"typesVersions\": {\n \"*\": {\n \"hmsvbplugin\": [\n \"./dist/HMSVBPlugin.d.ts\"\n ],\n \"hmseffectsplugin\": [\n \"./dist/HMSEffectsPlugin.d.ts\"\n ]\n }\n },\n \"exports\": {\n \".\": {\n \"import\": \"./dist/esm/index.js\",\n \"require\": \"./dist/cjs/index.js\",\n \"default\": \"./dist/esm/index.js\",\n \"types\": \"./dist/index.d.ts\"\n },\n \"./hmsvbplugin\": {\n \"import\": \"./dist/esm/HMSVBPlugin.js\",\n \"require\": \"./dist/cjs/HMSVBPlugin.js\",\n \"default\": \"./dist/esm/HMSVBPlugin.js\",\n \"types\": \"./dist/HMSVBPlugin.d.ts\"\n },\n \"./hmseffectsplugin\": {\n \"import\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"default\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"require\": \"./dist/cjs/HMSEffectsPlugin.js\",\n \"types\": \"./dist/HMSEffectsPlugin.d.ts\"\n }\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"https://github.com/100mslive/web-sdks.git\",\n \"directory\": \"packages/hms-virtual-background\"\n },\n \"files\": [\n \"dist\",\n \"src/tflite\",\n \"src/models\"\n ],\n \"scripts\": {\n \"start\": \"concurrently \\\"yarn dev\\\" \\\"yarn types\\\"\",\n \"dev\": \"node ../../scripts/dev\",\n \"build:only\": \"node ../../scripts/build\",\n \"build\": \"yarn build:only && yarn types:build\",\n \"types\": \"tsc -w\",\n \"types:build\": \"tsc -p tsconfig.json\",\n \"test\": \"jest --maxWorkers=1 --passWithNoTests\",\n \"lint\": \"eslint -c ../../.eslintrc .\",\n \"lint:fix\": \"yarn lint --fix\",\n \"prepare\": \"yarn build\",\n \"size\": \"size-limit\",\n \"analyze\": \"size-limit --why\",\n \"format\": \"prettier --write src/**/*.ts\"\n },\n \"peerDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.28\"\n },\n \"devDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.28\"\n },\n \"dependencies\": {\n \"@mediapipe/selfie_segmentation\": \"^0.1.1632777926\",\n \"@tensorflow-models/body-segmentation\": \"^1.0.1\",\n \"@tensorflow/tfjs-backend-webgl\": \"^3.3.0\",\n \"@tensorflow/tfjs-converter\": \"^3.19.0\",\n \"@tensorflow/tfjs-core\": \"^3.19.0\",\n \"@webassemblyjs/helper-wasm-bytecode\": \"1.11.1\",\n \"@webassemblyjs/wasm-gen\": \"1.11.1\",\n \"effects-sdk\": \"3.4.3\",\n \"gifuct-js\": \"^2.1.2\",\n \"wasm-check\": \"^2.0.2\"\n },\n \"eslintIgnore\": [\n \"tflite.js\",\n \"tflite-simd.js\",\n \"tflite.wasm\",\n \"tflite-simd.wasm\",\n \"defineTFLite.ts\",\n \"importing.test.ts\"\n ],\n \"gitHead\": \"c4d7367340117e66de386fc7aaa9b5e95abd19ed\"\n}\n", "export * from './HMSVirtualBackgroundPlugin';\nexport { HMSVBPlugin } from './HMSVBPlugin';\nexport { HMSVirtualBackgroundTypes } from './interfaces';\nexport type { HMSVirtualBackground } from './interfaces';\nexport { HMSEffectsPlugin } from './HMSEffectsPlugin';\nexport type { HMSEffectsBackground } from './HMSEffectsPlugin';\n", "/* eslint-disable complexity */\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport '@tensorflow/tfjs-backend-webgl';\nimport { loadTFLite } from './defineTFLite';\n\nconst TAG = 'VBProcessor';\nconst DEFAULT_DELAY = 33;\nconst pkg = require('../package.json');\n\nconst minVideoWidthForSharpening = 214;\nconst maxVideoWidthForSharpening = 855;\nconst minVideoHeightForSharpening = 120;\nconst maxVideoHeightForSharpening = 720;\n\nexport class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {\n background: string | HTMLImageElement;\n personMaskWidth: number;\n personMaskHeight: number;\n isVirtualBackground: boolean;\n backgroundImage: HTMLImageElement | null;\n backgroundVideo: HTMLVideoElement | null;\n backgroundType = 'none';\n loadModelCalled: boolean;\n blurValue: any;\n tfLite: any;\n tfLitePromise: any;\n modelName: string;\n\n input: HTMLCanvasElement | null;\n output: HTMLCanvasElement | null;\n outputCtx: CanvasRenderingContext2D | null;\n timerID: number;\n imageAspectRatio: number;\n\n personMaskPixelCount: number;\n personMask: ImageData;\n personMaskCanvas: HTMLCanvasElement;\n personMaskCtx: any;\n filters: any;\n enableSharpening?: boolean | false;\n\n gifFrames: any;\n gifFramesIndex: number;\n gifFrameImageData: any;\n tempGifCanvas: HTMLCanvasElement;\n tempGifContext: any;\n giflocalCount: number;\n\n constructor(background: string, enableSharpening = false) {\n this.background = background;\n this.enableSharpening = enableSharpening;\n\n this.backgroundImage = null;\n this.backgroundVideo = null;\n\n this.personMaskWidth = 256;\n this.personMaskHeight = 144;\n this.isVirtualBackground = false;\n this.blurValue = '10px';\n this.loadModelCalled = false;\n this.tfLite = null;\n this.modelName = 'landscape-segmentation';\n\n this.outputCtx = null;\n this.input = null;\n this.output = null;\n this.timerID = 0;\n this.imageAspectRatio = 1;\n\n this.personMaskPixelCount = this.personMaskWidth * this.personMaskHeight;\n this.personMask = new ImageData(this.personMaskWidth, this.personMaskHeight);\n this.personMaskCanvas = document.createElement('canvas');\n this.personMaskCanvas.width = this.personMaskWidth;\n this.personMaskCanvas.height = this.personMaskHeight;\n this.personMaskCtx = this.personMaskCanvas.getContext('2d');\n\n this.filters = {};\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n this.giflocalCount = 0;\n this.enableSharpening = enableSharpening;\n\n this.log(TAG, 'Virtual Background plugin created');\n this.setBackground(this.background);\n }\n\n async init(): Promise<void> {\n if (!this.loadModelCalled) {\n this.log(TAG, 'PREVIOUS LOADED MODEL IS ', this.tfLite);\n this.loadModelCalled = true;\n this.tfLitePromise = loadTFLite();\n this.tfLite = await this.tfLitePromise;\n } else {\n //either it is loading or loaded\n await this.tfLitePromise;\n }\n if (this.enableSharpening) {\n this.initSharpenFilter();\n }\n }\n\n /*\n @depreceated\n */\n isSupported(): boolean {\n //support chrome, firefox, edge TODO: check this\n return (\n navigator.userAgent.indexOf('Chrome') !== -1 ||\n navigator.userAgent.indexOf('Firefox') !== -1 ||\n navigator.userAgent.indexOf('Edg') !== -1 ||\n navigator.userAgent.indexOf('Edge') !== -1\n );\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return pkg.name;\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async setBackground(path?: string | HTMLImageElement | HTMLVideoElement) {\n if (path !== '') {\n if (path === 'none') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'none';\n this.backgroundType = 'none';\n this.isVirtualBackground = false;\n } else if (path === 'blur') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'blur';\n this.backgroundType = 'blur';\n this.isVirtualBackground = false;\n } else if (path instanceof HTMLImageElement) {\n this.log('setting background to image', path);\n const img = await this.setImage(path as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.isVirtualBackground = true;\n this.backgroundImage = img;\n this.backgroundType = 'image';\n }\n } else if (path instanceof HTMLVideoElement) {\n this.log('setting background to video', path);\n this.backgroundVideo = path as HTMLVideoElement;\n this.backgroundVideo.crossOrigin = 'anonymous';\n this.backgroundVideo.muted = true;\n this.backgroundVideo.loop = true;\n this.backgroundVideo.oncanplaythrough = async () => {\n if (this.backgroundVideo != null) {\n await this.backgroundVideo!.play();\n this.isVirtualBackground = true;\n this.backgroundType = 'video';\n }\n };\n } else {\n console.log('setting gif to background');\n this.gifFrames = await this.setGiF(path as string);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = 'gif';\n this.isVirtualBackground = true;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n\n stop(): void {\n if (this.isVirtualBackground) {\n this.backgroundImage?.removeAttribute('src');\n this.backgroundVideo?.removeAttribute('src');\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.loop = false;\n this.backgroundVideo = null;\n }\n }\n if (this.outputCtx) {\n this.outputCtx!.fillStyle = `rgb(0, 0, 0)`;\n this.outputCtx!.fillRect(0, 0, this.output!.width, this.output!.height);\n }\n\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.giflocalCount = 0;\n this.gifFramesIndex = 0;\n }\n\n processVideoFrame(\n input: HTMLCanvasElement,\n output: HTMLCanvasElement,\n skipProcessing?: boolean,\n ): Promise<void> | void {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n\n this.input = input;\n this.output = output;\n\n const ctx = output.getContext('2d');\n if (ctx!.canvas.width !== input.width) {\n ctx!.canvas.width = input.width;\n }\n if (ctx!.canvas.height !== input.height) {\n ctx!.canvas.height = input.height;\n }\n\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.width = input.width;\n this.backgroundVideo!.height = input.height;\n }\n\n this.outputCtx = ctx!;\n this.imageAspectRatio = input.width / input.height;\n if (this.imageAspectRatio <= 0) {\n throw new Error('Invalid input width/height');\n }\n\n const process = async () => {\n await this.runSegmentation(skipProcessing);\n };\n\n if (this.background === 'none' && !this.isVirtualBackground) {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n this.outputCtx!.drawImage(input, 0, 0, input.width, input.height);\n } else {\n process();\n }\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private setGiF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(tag: string, ...data: any[]) {\n console.info(tag, ...data);\n }\n\n private resizeInputData() {\n this.personMaskCtx!.drawImage(\n this.input,\n 0,\n 0,\n this.input!.width,\n this.input!.height,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n );\n\n const imageData = this.personMaskCtx!.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight);\n //\n const inputMemoryOffset = this.tfLite._getInputMemoryOffset() / 4;\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3] = imageData.data[i * 4] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 1] = imageData.data[i * 4 + 1] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 2] = imageData.data[i * 4 + 2] / 255;\n }\n }\n private infer(skipProcessing?: boolean) {\n if (!skipProcessing) {\n this.tfLite._runInference();\n }\n const outputMemoryOffset = this.tfLite._getOutputMemoryOffset() / 4;\n\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n if (this.modelName === 'meet') {\n const background = this.tfLite.HEAPF32[outputMemoryOffset + i * 2];\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i * 2 + 1];\n const shift = Math.max(background, person);\n const backgroundExp = Math.exp(background - shift);\n const personExp = Math.exp(person - shift);\n // Sets only the alpha component of each pixel.\n this.personMask.data[i * 4 + 3] = (255 * personExp) / (backgroundExp + personExp);\n } else if (this.modelName === 'landscape-segmentation') {\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i];\n this.personMask.data[i * 4 + 3] = 255 * person;\n }\n }\n\n this.personMaskCtx!.putImageData(this.personMask, 0, 0);\n }\n\n private postProcessing() {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n\n if (this.isVirtualBackground) {\n this.outputCtx!.filter = 'blur(4px)';\n } else {\n this.outputCtx!.filter = 'blur(8px)';\n }\n this.drawPersonMask();\n this.outputCtx!.globalCompositeOperation = 'source-in';\n this.outputCtx!.filter = 'none';\n // //Draw the foreground\n this.outputCtx!.drawImage(this.input!, 0, 0);\n\n if (\n this.enableSharpening &&\n this.output!.width > minVideoWidthForSharpening && // minimum and maximum resolution to enable sharpening filter\n this.output!.height > minVideoHeightForSharpening &&\n this.output!.width < maxVideoWidthForSharpening &&\n this.output!.height < maxVideoHeightForSharpening\n ) {\n this.sharpenFilter();\n }\n\n // //Draw the background\n this.drawSegmentedBackground();\n }\n\n private sharpenFilter() {\n // adding sharpening filter to each frame to improve edges and brightness\n // The basic idea is that you take the weighed sum of a rectangle of pixels from the source image and use that as the output value using convolution filter\n // It is applied intermediate output with black background and only mask data in frame\n // Filter currently used is 3 x 3 sharpening filter with values as shown:\n // [ 0, -1, 0,\n // -1, 5, -1,\n // 0, -1, 0 ]\n const outputImageData = this.outputCtx!.getImageData(0, 0, this.output!.width, this.output!.height);\n\n // filters you may try\n // [-1, -1, -1, -1, 9, -1, -1, -1, -1]\n //[0, -1, 0, -1, 5, -1, 0, -1, 0]\n const output = this.filters.convolute(outputImageData);\n\n this.outputCtx!.putImageData(output, 0, 0);\n }\n\n private drawPersonMask() {\n this.outputCtx!.drawImage(\n this.personMaskCanvas,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private drawSegmentedBackground() {\n this.outputCtx!.globalCompositeOperation = 'destination-over';\n this.outputCtx!.imageSmoothingEnabled = true;\n this.outputCtx!.imageSmoothingQuality = 'high';\n if (this.isVirtualBackground) {\n if (this.backgroundType === 'video' && this.backgroundVideo != null && this.backgroundVideo!.readyState >= 4) {\n this.fitVideoToBackground();\n } else if (this.backgroundType === 'image') {\n this.fitImageToBackground();\n } else if (this.backgroundType === 'gif') {\n if (this.giflocalCount > this.gifFrames[this.gifFramesIndex].delay / DEFAULT_DELAY) {\n this.gifFramesIndex++;\n if (this.gifFramesIndex >= this.gifFrames.length) {\n this.gifFramesIndex = 0;\n }\n this.giflocalCount = 0;\n } else {\n this.giflocalCount++;\n }\n this.fitGifToBackground();\n }\n } else {\n this.addBlurToBackground();\n }\n }\n\n private async runSegmentation(skipProcessing?: boolean) {\n if (this.tfLite) {\n // const start = performance.now();\n\n this.resizeInputData();\n await this.infer(skipProcessing);\n this.postProcessing();\n // const end = performance.now();\n // this.log(TAG,\"time taken\",end -start);\n }\n }\n\n private fitVideoToBackground() {\n this.fitData(this.backgroundVideo, this.backgroundVideo!.videoWidth, this.backgroundVideo!.videoHeight);\n }\n\n private fitImageToBackground() {\n this.fitData(this.backgroundImage, this.backgroundImage!.width, this.backgroundImage!.height);\n }\n\n private fitGifToBackground() {\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas!.width = dims.width;\n this.tempGifCanvas!.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n\n this.fitData(this.tempGifCanvas, this.gifFrameImageData!.width, this.gifFrameImageData!.height);\n }\n\n private fitData(data: any, dataWidth: number, dataHeight: number) {\n let inputWidth: any, inputHeight: any, xoffset: any, yoffset: any;\n if (dataWidth / dataHeight < this.imageAspectRatio) {\n inputWidth = dataWidth;\n inputHeight = dataWidth / this.imageAspectRatio;\n xoffset = 0;\n yoffset = (dataHeight - inputHeight) / 2;\n } else {\n inputHeight = dataHeight;\n inputWidth = dataHeight * this.imageAspectRatio;\n yoffset = 0;\n xoffset = (dataWidth - inputWidth) / 2;\n }\n this.outputCtx!.drawImage(\n data,\n xoffset,\n yoffset,\n inputWidth,\n inputHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private async addBlurToBackground() {\n let blurValue = '15px';\n if (this.input!.width <= 160) {\n blurValue = '5px';\n } else if (this.input!.width <= 320) {\n blurValue = '10px';\n } else if (this.input!.width <= 640) {\n blurValue = '15px';\n } else if (this.input!.width <= 960) {\n blurValue = '20px';\n } else if (this.input!.width <= 1280) {\n blurValue = '25px';\n } else if (this.input!.width <= 1920) {\n blurValue = '30px';\n }\n\n this.outputCtx!.filter = `blur(${blurValue})`;\n this.outputCtx!.drawImage(this.input!, 0, 0, this.output!.width, this.output!.height);\n }\n\n private initSharpenFilter(): any {\n this.filters.tmpCanvas = document.createElement('canvas');\n this.filters.tmpCtx = this.filters.tmpCanvas.getContext('2d');\n\n this.filters.createImageData = (w: number, h: number) => {\n return this.filters.tmpCtx.createImageData(w, h);\n };\n\n this.filters.convolute = (pixels: ImageData, weights = [0, -1, 0, -1, 5, -1, 0, -1, 0], opaque: boolean) => {\n const side = Math.round(Math.sqrt(weights.length));\n\n const halfSide = Math.floor(side / 2);\n const src = pixels.data;\n const sw = pixels.width;\n const sh = pixels.height;\n // pad output by the convolution matrix\n const w = sw;\n const h = sh;\n const output = this.filters.createImageData(w, h);\n const dst = output.data;\n // go through the destination image pixels\n const alphaFac = opaque ? 1 : 0;\n for (let y = 0; y < h; y = y + 1) {\n for (let x = 0; x < w; x = x + 1) {\n const dstOff = (y * w + x) * 4;\n\n if (src[dstOff + 3] === 0) {\n continue;\n } else if (x < w && y < h) {\n const sy = y;\n const sx = x;\n\n // calculate the weighed sum of the source image pixels that\n // fall under the convolution matrix\n let r = 0,\n g = 0,\n b = 0,\n a = 0;\n for (let cy = 0; cy < side; cy++) {\n for (let cx = 0; cx < side; cx++) {\n const scy = sy + cy - halfSide;\n const scx = sx + cx - halfSide;\n if (scy >= 0 && scy < sh && scx >= 0 && scx < sw) {\n const srcOff = (scy * sw + scx) * 4;\n const wt = weights[cy * side + cx];\n r += src[srcOff] * wt;\n g += src[srcOff + 1] * wt;\n b += src[srcOff + 2] * wt;\n a += src[srcOff + 3] * wt;\n }\n }\n }\n dst[dstOff] = r;\n dst[dstOff + 1] = g;\n dst[dstOff + 2] = b;\n dst[dstOff + 3] = a + alphaFac * (255 - a);\n }\n }\n }\n return output;\n };\n }\n}\n", "const pkg = require('../package.json');\nconst BASE_URL = `https://unpkg.com/${pkg.name}/src`;\nconst TAG = 'VBProcessor';\nconst TFLITE_JS_FILE = 'tflite/tflite.js';\nconst TFLITE_SIMD_JS_FILE = 'tflite/tflite-simd.js';\nconst MODEL_FILE_NAME = 'models/selfie_segmentation_landscape.tflite';\n\nconst loadScript = (src: string) => {\n return new Promise(function (resolve, reject) {\n const s = document.createElement('script');\n s.src = src;\n s.onload = resolve;\n s.onerror = reject;\n document.head.appendChild(s);\n });\n};\n\nconst loadTFLiteModel = async () => {\n let tfLite: any;\n let path = `${BASE_URL}/${TFLITE_SIMD_JS_FILE}`;\n await loadScript(path);\n try {\n //@ts-ignore\n tfLite = await createTFLiteSIMDModule();\n } catch {\n console.warn('SIMD not supported. You may experience poor virtual background effect.');\n path = `${BASE_URL}/${TFLITE_JS_FILE}`;\n await loadScript(path);\n // @ts-ignore\n tfLite = await createTFLiteModule();\n }\n return tfLite;\n};\n\nconst loadTFLite = async () => {\n const modelPath = `${BASE_URL}/${MODEL_FILE_NAME}`;\n const [tfLite, modelResponse] = await Promise.all([loadTFLiteModel(), fetch(modelPath)]);\n\n const model = await modelResponse.arrayBuffer();\n const modelBufferOffset = tfLite._getModelBufferMemoryOffset();\n tfLite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);\n tfLite._loadModel(model.byteLength);\n\n console.debug(TAG, 'Input memory offset:', tfLite._getInputMemoryOffset());\n console.debug(TAG, 'Input height:', tfLite._getInputHeight());\n console.debug(TAG, 'Input width:', tfLite._getInputWidth());\n console.debug(TAG, 'Input channels:', tfLite._getInputChannelCount());\n\n return tfLite;\n};\n\nexport { loadTFLite };\n", "/* eslint-disable complexity */\nimport { Results as MediaPipeResults, SelfieSegmentation } from '@mediapipe/selfie_segmentation';\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport { HMSBackgroundInput, HMSVirtualBackground, HMSVirtualBackgroundTypes } from './interfaces';\n\nexport class HMSVBPlugin implements HMSVideoPlugin {\n private TAG = '[HMSVBPlugin]';\n private background: HMSVirtualBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType: HMSVirtualBackgroundTypes = HMSVirtualBackgroundTypes.NONE;\n private segmentation!: SelfieSegmentation;\n private outputCanvas?: HTMLCanvasElement;\n private outputCtx?: CanvasRenderingContext2D | null;\n\n private gifFrames: any;\n private gifFramesIndex: number;\n private gifFrameImageData: any;\n private tempGifCanvas: HTMLCanvasElement;\n private tempGifContext: CanvasRenderingContext2D | null;\n private prevResults?: MediaPipeResults;\n private input?: HTMLCanvasElement;\n\n constructor(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n this.background = background;\n this.backgroundType = backgroundType;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n\n this.setBackground(this.background, this.backgroundType);\n this.log('Virtual background plugin initialised');\n }\n\n isSupported(): boolean {\n return this.checkSupport().isSupported;\n }\n\n isBlurSupported(): boolean {\n return 'filter' in CanvasRenderingContext2D.prototype;\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge', 'Safari'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return 'HMSVB';\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async init(): Promise<void> {\n if (!this.segmentation) {\n this.segmentation = new SelfieSegmentation({\n locateFile: (file: string) => {\n return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${file}`;\n },\n });\n this.segmentation.setOptions({ selfieMode: false, modelSelection: 1 });\n this.segmentation.onResults(this.handleResults);\n }\n }\n\n /**\n * For bgType HMSVirtualBackgroundTypes.IMAGE pass bg as an image element\n * For bgType HMSVirtualBackgroundTypes.VIDEO pass video\n * For bgType HMSVirtualBackgroundTypes.GIF pass the gif url\n * @param {HMSVirtualBackground} background\n * @param {HMSVirtualBackgroundTypes} backgroundType\n */\n async setBackground(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n if (!background) {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n this.prevResults = undefined;\n switch (backgroundType) {\n case HMSVirtualBackgroundTypes.NONE:\n case HMSVirtualBackgroundTypes.BLUR:\n this.background = background;\n this.backgroundType = backgroundType;\n break;\n case HMSVirtualBackgroundTypes.IMAGE:\n this.log('setting background to image', background);\n // eslint-disable-next-line no-case-declarations\n const img = await this.setImage(background as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.background = img;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n }\n break;\n case HMSVirtualBackgroundTypes.VIDEO:\n this.log('setting background to video', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as HTMLVideoElement;\n this.background.crossOrigin = 'anonymous';\n this.background.muted = true;\n this.background.loop = true;\n this.background.playsInline = true;\n this.background.oncanplaythrough = async () => {\n if (this.background && this.background instanceof HTMLVideoElement) {\n try {\n await this.background.play();\n this.backgroundType = HMSVirtualBackgroundTypes.VIDEO;\n } catch (e) {\n this.log('failed to play background', background);\n }\n }\n };\n break;\n case HMSVirtualBackgroundTypes.CANVAS:\n this.background = background;\n this.backgroundType = HMSVirtualBackgroundTypes.CANVAS;\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.log('setting gif to background', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as string;\n this.gifFrames = await this.loadGIF(this.background);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = HMSVirtualBackgroundTypes.GIF;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n break;\n default:\n this.log(\n `backgroundType did not match with any of the supported background types - ${HMSVirtualBackgroundTypes}`,\n );\n }\n }\n\n getBackground() {\n return this.background;\n }\n\n stop(): void {\n if (this.backgroundType !== HMSVirtualBackgroundTypes.BLUR && this.background !== HMSVirtualBackgroundTypes.NONE) {\n this.segmentation?.reset();\n }\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.background = HMSVirtualBackgroundTypes.NONE;\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n }\n\n async processVideoFrame(input: HTMLCanvasElement, output: HTMLCanvasElement, skipProcessing?: boolean) {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n this.input = input;\n output.width = input.width;\n output.height = input.height;\n this.outputCanvas = output;\n this.outputCtx = output.getContext('2d');\n if (skipProcessing && this.prevResults) {\n this.handleResults(this.prevResults);\n return;\n }\n if (this.backgroundType === HMSVirtualBackgroundTypes.NONE) {\n this.outputCtx?.drawImage(input, 0, 0, input.width, input.height);\n return;\n }\n await this.segmentation.send({ image: input });\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private handleResults = (results: MediaPipeResults) => {\n if (!this.outputCanvas || !this.outputCtx) {\n return;\n }\n this.outputCtx.save();\n this.outputCtx.clearRect(0, 0, this.outputCanvas.width, this.outputCanvas.height);\n switch (this.backgroundType) {\n case HMSVirtualBackgroundTypes.IMAGE:\n case HMSVirtualBackgroundTypes.CANVAS:\n case HMSVirtualBackgroundTypes.VIDEO:\n this.renderBackground(results, this.background as HMSBackgroundInput);\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.renderGIF(results);\n break;\n case HMSVirtualBackgroundTypes.BLUR:\n this.renderBlur(results);\n break;\n }\n this.outputCtx.restore();\n this.prevResults = results;\n };\n\n private loadGIF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(...data: any[]) {\n console.debug(this.TAG, ...data);\n }\n\n private renderBackground = (results: MediaPipeResults, background: HMSBackgroundInput) => {\n if (\n !this.input ||\n !this.outputCanvas ||\n !this.outputCtx ||\n this.backgroundType === HMSVirtualBackgroundTypes.NONE ||\n this.backgroundType === HMSVirtualBackgroundTypes.BLUR\n ) {\n return;\n }\n this.outputCtx.filter = 'none';\n this.outputCtx.imageSmoothingEnabled = true;\n this.outputCtx.imageSmoothingQuality = 'high';\n // Only overwrite existing pixels.\n this.outputCtx.globalCompositeOperation = 'source-out';\n const bgWidth = background instanceof HTMLVideoElement ? background.videoWidth : background.width;\n const bgHeight = background instanceof HTMLVideoElement ? background.videoHeight : background.height;\n\n this.outputCtx.drawImage(\n background,\n 0,\n 0,\n bgWidth,\n bgHeight,\n 0,\n 0,\n this.outputCanvas.width,\n this.outputCanvas.height,\n );\n this.outputCtx.globalCompositeOperation = 'destination-out';\n this.outputCtx.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n // Only overwrite missing pixels.\n this.outputCtx.globalCompositeOperation = 'destination-atop';\n this.outputCtx.drawImage(this.input, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n };\n\n private renderBlur(results: MediaPipeResults) {\n if (!this.outputCanvas || !this.outputCtx || this.backgroundType !== HMSVirtualBackgroundTypes.BLUR) {\n return;\n }\n this.outputCtx!.filter = 'none';\n this.outputCtx!.globalCompositeOperation = 'source-out';\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.globalCompositeOperation = 'destination-atop';\n this.outputCtx?.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.filter = `blur(${Math.floor(this.outputCanvas.width / 160) * 5}px)`;\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n }\n\n private renderGIF(results: MediaPipeResults) {\n if (\n !this.outputCanvas ||\n !this.outputCtx ||\n !this.tempGifContext ||\n this.backgroundType !== HMSVirtualBackgroundTypes.GIF\n ) {\n return;\n }\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas.width = dims.width;\n this.tempGifCanvas.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n this.gifFramesIndex = (this.gifFramesIndex + 1) % this.gifFrames.length;\n this.renderBackground(results, this.tempGifCanvas);\n }\n}\n\nexport { HMSVirtualBackgroundTypes };\n", "export type HMSBackgroundInput = HTMLImageElement | HTMLVideoElement | HTMLCanvasElement;\nexport enum HMSVirtualBackgroundTypes {\n BLUR = 'blur',\n NONE = 'none',\n GIF = 'gif',\n IMAGE = 'image',\n VIDEO = 'video',\n CANVAS = 'canvas',\n}\n\nexport type HMSVirtualBackground =\n | string\n | HMSVirtualBackgroundTypes.BLUR\n | HMSVirtualBackgroundTypes.NONE\n | HMSBackgroundInput;\n", "import { tsvb } from 'effects-sdk';\nimport { HMSMediaStreamPlugin } from '@100mslive/hms-video-store';\nimport { EFFECTS_SDK_ASSETS } from './constants';\nimport { HMSVirtualBackgroundTypes } from './interfaces';\n\nexport type HMSEffectsBackground = string | MediaStream | MediaStreamTrack | HTMLVideoElement;\n\nexport class HMSEffectsPlugin implements HMSMediaStreamPlugin {\n private effects: tsvb;\n // Ranges from 0 to 1, inclusive\n private blurAmount = 0;\n private background: HMSEffectsBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType = HMSVirtualBackgroundTypes.NONE;\n private preset: 'balanced' | 'quality' = 'balanced';\n private initialised = false;\n private intervalId: NodeJS.Timer | null = null;\n private onInit;\n private onResolutionChangeCallback?: (width: number, height: number) => void;\n private canvas: HTMLCanvasElement;\n\n constructor(effectsSDKKey: string, onInit?: () => void) {\n this.effects = new tsvb(effectsSDKKey);\n this.onInit = onInit;\n this.effects.config({\n sdk_url: EFFECTS_SDK_ASSETS,\n models: {\n colorcorrector: '',\n facedetector: '',\n lowlighter: '',\n },\n wasmPaths: {\n 'ort-wasm.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm.wasm`,\n 'ort-wasm-simd.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd.wasm`,\n },\n });\n this.canvas = document.createElement('canvas');\n this.effects.onError(err => {\n // currently logging info type messages as well\n if (!err.type || err.type === 'error') {\n console.error('[HMSEffectsPlugin]', err);\n }\n });\n this.effects.cache();\n this.effects.onReady = () => {\n if (this.effects) {\n this.initialised = true;\n this.onInit?.();\n this.effects.run();\n this.effects.setBackgroundFitMode('fill');\n this.effects.setSegmentationPreset(this.preset);\n this.applyEffect();\n }\n };\n }\n\n getName(): string {\n return 'HMSEffects';\n }\n\n private executeAfterInit(callback: () => void) {\n if (this.initialised) {\n callback();\n }\n\n if (this.intervalId !== null) {\n clearInterval(this.intervalId);\n }\n this.intervalId = setInterval(() => {\n if (this.initialised) {\n clearInterval(this.intervalId!);\n callback();\n }\n }, 100);\n }\n\n removeBlur() {\n this.blurAmount = 0;\n this.executeAfterInit(() => {\n this.effects.clearBlur();\n });\n }\n\n removeBackground() {\n this.background = '';\n this.executeAfterInit(() => {\n this.effects.clearBackground();\n });\n }\n\n /**\n * @param blur ranges between 0 and 1\n */\n setBlur(blur: number) {\n this.blurAmount = blur;\n this.backgroundType = HMSVirtualBackgroundTypes.BLUR;\n this.removeBackground();\n this.executeAfterInit(() => {\n this.effects.setBlur(this.blurAmount);\n });\n }\n\n /**\n * @param preset can be 'quality' or 'balanced'. The 'quality' preset has better quality but higher CPU usage than 'balanced'\n */\n async setPreset(preset: 'quality' | 'balanced') {\n this.preset = preset;\n return new Promise((resolve, reject) => {\n this.executeAfterInit(() => {\n this.effects.setSegmentationPreset(this.preset).then(resolve).catch(reject);\n });\n });\n }\n\n onResolutionChange(callback: (width: number, height: number) => void) {\n this.onResolutionChangeCallback = callback;\n }\n\n getPreset() {\n return this.preset;\n }\n\n removeEffects() {\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.removeBackground();\n this.removeBlur();\n }\n\n setBackground(url: HMSEffectsBackground) {\n this.background = url;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n this.removeBlur();\n this.executeAfterInit(() => {\n this.effects.setBackground(this.background);\n });\n }\n\n getBlurAmount() {\n return this.blurAmount;\n }\n\n getBackground() {\n return this.background || this.backgroundType;\n }\n\n private updateCanvas(stream: MediaStream) {\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.canvas.width = width!;\n this.canvas.height = height!;\n this.effects.useStream(stream);\n this.effects.toCanvas(this.canvas);\n }\n\n apply(stream: MediaStream): MediaStream {\n this.effects.clear();\n this.applyEffect();\n this.effects.onChangeInputResolution(() => {\n this.updateCanvas(stream);\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.onResolutionChangeCallback?.(width!, height!);\n });\n this.updateCanvas(stream);\n return this.canvas.captureStream(30) || stream;\n }\n\n stop() {\n this.removeEffects();\n this.executeAfterInit(() => {\n this.effects.stop();\n });\n }\n\n private applyEffect() {\n if (this.blurAmount) {\n this.setBlur(this.blurAmount);\n } else if (this.background) {\n this.setBackground(this.background);\n }\n }\n}\n", "export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/3.4.3/';\n"],
5
5
  "mappings": "irBAAA,IAAAA,EAAAC,EAAA,CAAAC,GAAAC,KAAA,CAAAA,GAAA,SACE,QAAW,UACX,QAAW,MACX,KAAQ,oCACR,OAAU,QACV,OAAU,oBACV,KAAQ,oBACR,QAAW,kBACX,cAAiB,CACf,IAAK,CACH,YAAe,CACb,yBACF,EACA,iBAAoB,CAClB,8BACF,CACF,CACF,EACA,QAAW,CACT,IAAK,CACH,OAAU,sBACV,QAAW,sBACX,QAAW,sBACX,MAAS,mBACX,EACA,gBAAiB,CACf,OAAU,4BACV,QAAW,4BACX,QAAW,4BACX,MAAS,yBACX,EACA,qBAAsB,CACpB,OAAU,iCACV,QAAW,iCACX,QAAW,iCACX,MAAS,8BACX,CACF,EACA,WAAc,CACZ,KAAQ,MACR,IAAO,4CACP,UAAa,iCACf,EACA,MAAS,CACP,OACA,aACA,YACF,EACA,QAAW,CACT,MAAS,uCACT,IAAO,yBACP,aAAc,2BACd,MAAS,sCACT,MAAS,SACT,cAAe,uBACf,KAAQ,wCACR,KAAQ,8BACR,WAAY,kBACZ,QAAW,aACX,KAAQ,aACR,QAAW,mBACX,OAAU,8BACZ,EACA,iBAAoB,CAClB,6BAA8B,SAChC,EACA,gBAAmB,CACjB,6BAA8B,SAChC,EACA,aAAgB,CACd,iCAAkC,kBAClC,uCAAwC,SACxC,iCAAkC,SAClC,6BAA8B,UAC9B,wBAAyB,UACzB,sCAAuC,SACvC,0BAA2B,SAC3B,cAAe,QACf,YAAa,SACb,aAAc,QAChB,EACA,aAAgB,CACd,YACA,iBACA,cACA,mBACA,kBACA,mBACF,EACA,QAAW,0CACb,IC1FA,IAAAC,GAAA,GAAAC,EAAAD,GAAA,sBAAAE,EAAA,gBAAAC,EAAA,+BAAAC,EAAA,8BAAAC,IAAA,eAAAC,GAAAN,ICCA,IAAAO,EAA2C,qBAC3CC,EAKO,sCACPC,GAAO,0CCRP,IAAMC,GAAM,IACNC,EAAW,qBAAqBD,GAAI,IAAI,OACxCE,EAAM,cACNC,GAAiB,mBACjBC,GAAsB,wBACtBC,GAAkB,8CAElBC,EAAcC,GACX,IAAI,QAAQ,SAAUC,EAASC,EAAQ,CAC5C,IAAMC,EAAI,SAAS,cAAc,QAAQ,EACzCA,EAAE,IAAMH,EACRG,EAAE,OAASF,EACXE,EAAE,QAAUD,EACZ,SAAS,KAAK,YAAYC,CAAC,CAC7B,CAAC,EAGGC,GAAkB,IAAYC,EAAA,wBAClC,IAAIC,EACAC,EAAO,GAAGb,CAAQ,IAAIG,EAAmB,GAC7C,MAAME,EAAWQ,CAAI,EACrB,GAAI,CAEFD,EAAS,MAAM,uBAAuB,CACxC,OAAQ,GACN,QAAQ,KAAK,wEAAwE,EACrFC,EAAO,GAAGb,CAAQ,IAAIE,EAAc,GACpC,MAAMG,EAAWQ,CAAI,EAErBD,EAAS,MAAM,mBAAmB,CACpC,CACA,OAAOA,CACT,GAEME,EAAa,IAAYH,EAAA,wBAC7B,IAAMI,EAAY,GAAGf,CAAQ,IAAII,EAAe,GAC1C,CAACQ,EAAQI,CAAa,EAAI,MAAM,QAAQ,IAAI,CAACN,GAAgB,EAAG,MAAMK,CAAS,CAAC,CAAC,EAEjFE,EAAQ,MAAMD,EAAc,YAAY,EACxCE,EAAoBN,EAAO,4BAA4B,EAC7D,OAAAA,EAAO,OAAO,IAAI,IAAI,WAAWK,CAAK,EAAGC,CAAiB,EAC1DN,EAAO,WAAWK,EAAM,UAAU,EAElC,QAAQ,MAAMhB,EAAK,uBAAwBW,EAAO,sBAAsB,CAAC,EACzE,QAAQ,MAAMX,EAAK,gBAAiBW,EAAO,gBAAgB,CAAC,EAC5D,QAAQ,MAAMX,EAAK,eAAgBW,EAAO,eAAe,CAAC,EAC1D,QAAQ,MAAMX,EAAK,kBAAmBW,EAAO,sBAAsB,CAAC,EAE7DA,CACT,GDtCA,IAAMO,EAAM,cACNC,GAAgB,GAChBC,GAAM,IAENC,GAA6B,IAC7BC,GAA6B,IAC7BC,GAA8B,IAC9BC,GAA8B,IAEvBC,EAAN,KAA2D,CAkChE,YAAYC,EAAoBC,EAAmB,GAAO,CA3B1D,oBAAiB,OA4Bf,KAAK,WAAaD,EAClB,KAAK,iBAAmBC,EAExB,KAAK,gBAAkB,KACvB,KAAK,gBAAkB,KAEvB,KAAK,gBAAkB,IACvB,KAAK,iBAAmB,IACxB,KAAK,oBAAsB,GAC3B,KAAK,UAAY,OACjB,KAAK,gBAAkB,GACvB,KAAK,OAAS,KACd,KAAK,UAAY,yBAEjB,KAAK,UAAY,KACjB,KAAK,MAAQ,KACb,KAAK,OAAS,KACd,KAAK,QAAU,EACf,KAAK,iBAAmB,EAExB,KAAK,qBAAuB,KAAK,gBAAkB,KAAK,iBACxD,KAAK,WAAa,IAAI,UAAU,KAAK,gBAAiB,KAAK,gBAAgB,EAC3E,KAAK,iBAAmB,SAAS,cAAc,QAAQ,EACvD,KAAK,iBAAiB,MAAQ,KAAK,gBACnC,KAAK,iBAAiB,OAAS,KAAK,iBACpC,KAAK,cAAgB,KAAK,iBAAiB,WAAW,IAAI,EAE1D,KAAK,QAAU,CAAC,EAChB,KAAK,UAAY,KACjB,KAAK,eAAiB,EACtB,KAAK,kBAAoB,KACzB,KAAK,cAAgB,SAAS,cAAc,QAAQ,EACpD,KAAK,eAAiB,KAAK,cAAc,WAAW,IAAI,EACxD,KAAK,cAAgB,EACrB,KAAK,iBAAmBA,EAExB,KAAK,IAAIT,EAAK,mCAAmC,EACjD,KAAK,cAAc,KAAK,UAAU,CACpC,CAEM,MAAsB,QAAAU,EAAA,sBACrB,KAAK,gBAOR,MAAM,KAAK,eANX,KAAK,IAAIV,EAAK,4BAA6B,KAAK,MAAM,EACtD,KAAK,gBAAkB,GACvB,KAAK,cAAgBW,EAAW,EAChC,KAAK,OAAS,MAAM,KAAK,eAKvB,KAAK,kBACP,KAAK,kBAAkB,CAE3B,GAKA,aAAuB,CAErB,OACE,UAAU,UAAU,QAAQ,QAAQ,IAAM,IAC1C,UAAU,UAAU,QAAQ,SAAS,IAAM,IAC3C,UAAU,UAAU,QAAQ,KAAK,IAAM,IACvC,UAAU,UAAU,QAAQ,MAAM,IAAM,EAE5C,CAEA,cAAuC,CACrC,IAAMC,EAAgB,CAAC,EACvB,MAAI,CAAC,SAAU,UAAW,MAAO,MAAM,EAAE,KAAKC,GAAS,UAAU,UAAU,QAAQA,CAAK,IAAM,EAAE,EAC9FD,EAAc,YAAc,IAE5BA,EAAc,YAAc,GAC5BA,EAAc,QAAU,4BAA0B,uBAClDA,EAAc,OAAS,8CAGlBA,CACT,CAEA,SAAkB,CAChB,OAAOV,GAAI,IACb,CAEA,eAAoC,CAClC,OAAO,qBAAmB,SAC5B,CAEM,cAAcY,EAAqD,QAAAJ,EAAA,sBACvE,GAAII,IAAS,GACX,GAAIA,IAAS,OACX,KAAK,IAAId,EAAK,0BAA2Bc,CAAI,EAC7C,KAAK,WAAa,OAClB,KAAK,eAAiB,OACtB,KAAK,oBAAsB,WAClBA,IAAS,OAClB,KAAK,IAAId,EAAK,0BAA2Bc,CAAI,EAC7C,KAAK,WAAa,OAClB,KAAK,eAAiB,OACtB,KAAK,oBAAsB,WAClBA,aAAgB,iBAAkB,CAC3C,KAAK,IAAI,8BAA+BA,CAAI,EAC5C,IAAMC,EAAM,MAAM,KAAK,SAASD,CAAwB,EACxD,GAAI,CAACC,GAAO,CAACA,EAAI,UAAY,CAACA,EAAI,cAChC,MAAM,IAAI,MAAM,yEAAyE,EAEzF,KAAK,oBAAsB,GAC3B,KAAK,gBAAkBA,EACvB,KAAK,eAAiB,OAE1B,SAAWD,aAAgB,iBACzB,KAAK,IAAI,8BAA+BA,CAAI,EAC5C,KAAK,gBAAkBA,EACvB,KAAK,gBAAgB,YAAc,YACnC,KAAK,gBAAgB,MAAQ,GAC7B,KAAK,gBAAgB,KAAO,GAC5B,KAAK,gBAAgB,iBAAmB,IAAYJ,EAAA,sBAC9C,KAAK,iBAAmB,OAC1B,MAAM,KAAK,gBAAiB,KAAK,EACjC,KAAK,oBAAsB,GAC3B,KAAK,eAAiB,QAE1B,WAEA,QAAQ,IAAI,2BAA2B,EACvC,KAAK,UAAY,MAAM,KAAK,OAAOI,CAAc,EAC7C,KAAK,WAAa,MAAQ,KAAK,UAAU,OAAS,EACpD,KAAK,eAAiB,MACtB,KAAK,oBAAsB,OAE3B,OAAM,IAAI,MAAM,8EAA8E,MAIlG,OAAM,IAAI,MAAM,8EAA8E,CAElG,GAEA,MAAa,CAlMf,IAAAE,EAAAC,EAmMQ,KAAK,uBACPD,EAAA,KAAK,kBAAL,MAAAA,EAAsB,gBAAgB,QACtCC,EAAA,KAAK,kBAAL,MAAAA,EAAsB,gBAAgB,OAClC,KAAK,iBAAmB,UAC1B,KAAK,gBAAiB,KAAO,GAC7B,KAAK,gBAAkB,OAGvB,KAAK,YACP,KAAK,UAAW,UAAY,eAC5B,KAAK,UAAW,SAAS,EAAG,EAAG,KAAK,OAAQ,MAAO,KAAK,OAAQ,MAAM,GAIxE,KAAK,kBAAoB,KACzB,KAAK,UAAY,KACjB,KAAK,cAAgB,EACrB,KAAK,eAAiB,CACxB,CAEA,kBACEC,EACAC,EACAC,EACsB,CACtB,GAAI,CAACF,GAAS,CAACC,EACb,MAAM,IAAI,MAAM,6BAA6B,EAG/C,KAAK,MAAQD,EACb,KAAK,OAASC,EAEd,IAAME,EAAMF,EAAO,WAAW,IAAI,EAelC,GAdIE,EAAK,OAAO,QAAUH,EAAM,QAC9BG,EAAK,OAAO,MAAQH,EAAM,OAExBG,EAAK,OAAO,SAAWH,EAAM,SAC/BG,EAAK,OAAO,OAASH,EAAM,QAGzB,KAAK,iBAAmB,UAC1B,KAAK,gBAAiB,MAAQA,EAAM,MACpC,KAAK,gBAAiB,OAASA,EAAM,QAGvC,KAAK,UAAYG,EACjB,KAAK,iBAAmBH,EAAM,MAAQA,EAAM,OACxC,KAAK,kBAAoB,EAC3B,MAAM,IAAI,MAAM,4BAA4B,EAG9C,IAAMI,EAAU,IAAYZ,EAAA,sBAC1B,MAAM,KAAK,gBAAgBU,CAAc,CAC3C,GAEI,KAAK,aAAe,QAAU,CAAC,KAAK,qBACtC,KAAK,UAAW,yBAA2B,OAC3C,KAAK,UAAW,OAAS,OACzB,KAAK,UAAW,UAAUF,EAAO,EAAG,EAAGA,EAAM,MAAOA,EAAM,MAAM,GAEhEI,EAAQ,CAEZ,CAEc,SAASC,EAAuC,QAAAb,EAAA,sBAC5D,OAAAa,EAAM,YAAc,YACb,IAAI,QAAQ,CAACC,EAASC,IAAW,CACtCF,EAAM,OAAS,IAAMC,EAAQD,CAAK,EAClCA,EAAM,QAAUE,CAClB,CAAC,CACH,GAEQ,OAAOC,EAA2B,CACxC,OAAO,MAAMA,CAAG,EACb,KAAKC,GAAQA,EAAK,YAAY,CAAC,EAC/B,KAAKC,MAAQ,YAASA,CAAI,CAAC,EAC3B,KAAKC,MACG,oBAAiBA,EAAK,EAAI,CAClC,CACL,CAEQ,IAAIC,KAAgBC,EAAa,CACvC,QAAQ,KAAKD,EAAK,GAAGC,CAAI,CAC3B,CAEQ,iBAAkB,CACxB,KAAK,cAAe,UAClB,KAAK,MACL,EACA,EACA,KAAK,MAAO,MACZ,KAAK,MAAO,OACZ,EACA,EACA,KAAK,gBACL,KAAK,gBACP,EAEA,IAAMC,EAAY,KAAK,cAAe,aAAa,EAAG,EAAG,KAAK,gBAAiB,KAAK,gBAAgB,EAE9FC,EAAoB,KAAK,OAAO,sBAAsB,EAAI,EAChE,QAAS,EAAI,EAAG,EAAI,KAAK,qBAAsB,IAC7C,KAAK,OAAO,QAAQA,EAAoB,EAAI,CAAC,EAAID,EAAU,KAAK,EAAI,CAAC,EAAI,IACzE,KAAK,OAAO,QAAQC,EAAoB,EAAI,EAAI,CAAC,EAAID,EAAU,KAAK,EAAI,EAAI,CAAC,EAAI,IACjF,KAAK,OAAO,QAAQC,EAAoB,EAAI,EAAI,CAAC,EAAID,EAAU,KAAK,EAAI,EAAI,CAAC,EAAI,GAErF,CACQ,MAAMZ,EAA0B,CACjCA,GACH,KAAK,OAAO,cAAc,EAE5B,IAAMc,EAAqB,KAAK,OAAO,uBAAuB,EAAI,EAElE,QAAS,EAAI,EAAG,EAAI,KAAK,qBAAsB,IAC7C,GAAI,KAAK,YAAc,OAAQ,CAC7B,IAAM1B,EAAa,KAAK,OAAO,QAAQ0B,EAAqB,EAAI,CAAC,EAC3DC,EAAS,KAAK,OAAO,QAAQD,EAAqB,EAAI,EAAI,CAAC,EAC3DE,EAAQ,KAAK,IAAI5B,EAAY2B,CAAM,EACnCE,EAAgB,KAAK,IAAI7B,EAAa4B,CAAK,EAC3CE,EAAY,KAAK,IAAIH,EAASC,CAAK,EAEzC,KAAK,WAAW,KAAK,EAAI,EAAI,CAAC,EAAK,IAAME,GAAcD,EAAgBC,EACzE,SAAW,KAAK,YAAc,yBAA0B,CACtD,IAAMH,EAAS,KAAK,OAAO,QAAQD,EAAqB,CAAC,EACzD,KAAK,WAAW,KAAK,EAAI,EAAI,CAAC,EAAI,IAAMC,CAC1C,CAGF,KAAK,cAAe,aAAa,KAAK,WAAY,EAAG,CAAC,CACxD,CAEQ,gBAAiB,CACvB,KAAK,UAAW,yBAA2B,OAC3C,KAAK,UAAW,OAAS,OAErB,KAAK,oBACP,KAAK,UAAW,OAAS,YAEzB,KAAK,UAAW,OAAS,YAE3B,KAAK,eAAe,EACpB,KAAK,UAAW,yBAA2B,YAC3C,KAAK,UAAW,OAAS,OAEzB,KAAK,UAAW,UAAU,KAAK,MAAQ,EAAG,CAAC,EAGzC,KAAK,kBACL,KAAK,OAAQ,MAAQhC,IACrB,KAAK,OAAQ,OAASE,IACtB,KAAK,OAAQ,MAAQD,IACrB,KAAK,OAAQ,OAASE,IAEtB,KAAK,cAAc,EAIrB,KAAK,wBAAwB,CAC/B,CAEQ,eAAgB,CAQtB,IAAMiC,EAAkB,KAAK,UAAW,aAAa,EAAG,EAAG,KAAK,OAAQ,MAAO,KAAK,OAAQ,MAAM,EAK5FpB,EAAS,KAAK,QAAQ,UAAUoB,CAAe,EAErD,KAAK,UAAW,aAAapB,EAAQ,EAAG,CAAC,CAC3C,CAEQ,gBAAiB,CACvB,KAAK,UAAW,UACd,KAAK,iBACL,EACA,EACA,KAAK,gBACL,KAAK,iBACL,EACA,EACA,KAAK,OAAQ,MACb,KAAK,OAAQ,MACf,CACF,CAEQ,yBAA0B,CAChC,KAAK,UAAW,yBAA2B,mBAC3C,KAAK,UAAW,sBAAwB,GACxC,KAAK,UAAW,sBAAwB,OACpC,KAAK,oBACH,KAAK,iBAAmB,SAAW,KAAK,iBAAmB,MAAQ,KAAK,gBAAiB,YAAc,EACzG,KAAK,qBAAqB,EACjB,KAAK,iBAAmB,QACjC,KAAK,qBAAqB,EACjB,KAAK,iBAAmB,QAC7B,KAAK,cAAgB,KAAK,UAAU,KAAK,cAAc,EAAE,MAAQlB,IACnE,KAAK,iBACD,KAAK,gBAAkB,KAAK,UAAU,SACxC,KAAK,eAAiB,GAExB,KAAK,cAAgB,GAErB,KAAK,gBAEP,KAAK,mBAAmB,GAG1B,KAAK,oBAAoB,CAE7B,CAEc,gBAAgBmB,EAA0B,QAAAV,EAAA,sBAClD,KAAK,SAGP,KAAK,gBAAgB,EACrB,MAAM,KAAK,MAAMU,CAAc,EAC/B,KAAK,eAAe,EAIxB,GAEQ,sBAAuB,CAC7B,KAAK,QAAQ,KAAK,gBAAiB,KAAK,gBAAiB,WAAY,KAAK,gBAAiB,WAAW,CACxG,CAEQ,sBAAuB,CAC7B,KAAK,QAAQ,KAAK,gBAAiB,KAAK,gBAAiB,MAAO,KAAK,gBAAiB,MAAM,CAC9F,CAEQ,oBAAqB,CAC3B,GAAI,KAAK,mBAAqB,KAAM,CAClC,IAAMoB,EAAO,KAAK,UAAU,KAAK,cAAc,EAAE,KACjD,KAAK,cAAe,MAAQA,EAAK,MACjC,KAAK,cAAe,OAASA,EAAK,OAClC,KAAK,kBAAoB,KAAK,eAAe,gBAAgBA,EAAK,MAAOA,EAAK,MAAM,CACtF,CAEA,KAAK,kBAAkB,KAAK,IAAI,KAAK,UAAU,KAAK,cAAc,EAAE,KAAK,EACzE,KAAK,eAAe,aAAa,KAAK,kBAAmB,EAAG,CAAC,EAE7D,KAAK,QAAQ,KAAK,cAAe,KAAK,kBAAmB,MAAO,KAAK,kBAAmB,MAAM,CAChG,CAEQ,QAAQT,EAAWU,EAAmBC,EAAoB,CAChE,IAAIC,EAAiBC,EAAkBC,EAAcC,EACjDL,EAAYC,EAAa,KAAK,kBAChCC,EAAaF,EACbG,EAAcH,EAAY,KAAK,iBAC/BI,EAAU,EACVC,GAAWJ,EAAaE,GAAe,IAEvCA,EAAcF,EACdC,EAAaD,EAAa,KAAK,iBAC/BI,EAAU,EACVD,GAAWJ,EAAYE,GAAc,GAEvC,KAAK,UAAW,UACdZ,EACAc,EACAC,EACAH,EACAC,EACA,EACA,EACA,KAAK,OAAQ,MACb,KAAK,OAAQ,MACf,CACF,CAEc,qBAAsB,QAAAlC,EAAA,sBAClC,IAAIqC,EAAY,OACZ,KAAK,MAAO,OAAS,IACvBA,EAAY,MACH,KAAK,MAAO,OAAS,IAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,IAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,IAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,KAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,OAC9BA,EAAY,QAGd,KAAK,UAAW,OAAS,QAAQA,CAAS,IAC1C,KAAK,UAAW,UAAU,KAAK,MAAQ,EAAG,EAAG,KAAK,OAAQ,MAAO,KAAK,OAAQ,MAAM,CACtF,GAEQ,mBAAyB,CAC/B,KAAK,QAAQ,UAAY,SAAS,cAAc,QAAQ,EACxD,KAAK,QAAQ,OAAS,KAAK,QAAQ,UAAU,WAAW,IAAI,EAE5D,KAAK,QAAQ,gBAAkB,CAACC,EAAWC,IAClC,KAAK,QAAQ,OAAO,gBAAgBD,EAAGC,CAAC,EAGjD,KAAK,QAAQ,UAAY,CAACC,EAAmBC,EAAU,CAAC,EAAG,GAAI,EAAG,GAAI,EAAG,GAAI,EAAG,GAAI,CAAC,EAAGC,IAAoB,CAC1G,IAAMC,EAAO,KAAK,MAAM,KAAK,KAAKF,EAAQ,MAAM,CAAC,EAE3CG,EAAW,KAAK,MAAMD,EAAO,CAAC,EAC9BE,EAAML,EAAO,KACbM,EAAKN,EAAO,MACZO,EAAKP,EAAO,OAEZF,EAAIQ,EACJP,EAAIQ,EACJtC,EAAS,KAAK,QAAQ,gBAAgB6B,EAAGC,CAAC,EAC1CS,EAAMvC,EAAO,KAEbwC,EAAWP,EAAS,EAAI,EAC9B,QAASQ,EAAI,EAAGA,EAAIX,EAAGW,EAAIA,EAAI,EAC7B,QAASC,EAAI,EAAGA,EAAIb,EAAGa,EAAIA,EAAI,EAAG,CAChC,IAAMC,GAAUF,EAAIZ,EAAIa,GAAK,EAE7B,GAAIN,EAAIO,EAAS,CAAC,IAAM,GAEbD,EAAIb,GAAKY,EAAIX,EAAG,CACzB,IAAMc,EAAKH,EACLI,EAAKH,EAIPI,EAAI,EACNC,EAAI,EACJC,EAAI,EACJC,EAAI,EACN,QAASC,EAAK,EAAGA,EAAKhB,EAAMgB,IAC1B,QAASC,EAAK,EAAGA,EAAKjB,EAAMiB,IAAM,CAChC,IAAMC,EAAMR,EAAKM,EAAKf,EAChBkB,EAAMR,EAAKM,EAAKhB,EACtB,GAAIiB,GAAO,GAAKA,EAAMd,GAAMe,GAAO,GAAKA,EAAMhB,EAAI,CAChD,IAAMiB,GAAUF,EAAMf,EAAKgB,GAAO,EAC5BE,EAAKvB,EAAQkB,EAAKhB,EAAOiB,CAAE,EACjCL,GAAKV,EAAIkB,CAAM,EAAIC,EACnBR,GAAKX,EAAIkB,EAAS,CAAC,EAAIC,EACvBP,GAAKZ,EAAIkB,EAAS,CAAC,EAAIC,EACvBN,GAAKb,EAAIkB,EAAS,CAAC,EAAIC,CACzB,CACF,CAEFhB,EAAII,CAAM,EAAIG,EACdP,EAAII,EAAS,CAAC,EAAII,EAClBR,EAAII,EAAS,CAAC,EAAIK,EAClBT,EAAII,EAAS,CAAC,EAAIM,EAAIT,GAAY,IAAMS,EAC1C,CACF,CAEF,OAAOjD,CACT,CACF,CACF,EE1iBA,IAAAwD,EAAgE,0CAChEC,EAA2C,qBAC3CC,EAKO,sCCPA,IAAKC,OACVA,EAAA,KAAO,OACPA,EAAA,KAAO,OACPA,EAAA,IAAM,MACNA,EAAA,MAAQ,QACRA,EAAA,MAAQ,QACRA,EAAA,OAAS,SANCA,OAAA,IDUL,IAAMC,EAAN,KAA4C,CAgBjD,YAAYC,EAAkCC,EAA2C,CAfzF,KAAQ,IAAM,gBACd,KAAQ,kBACR,KAAQ,sBAqLR,KAAQ,cAAiBC,GAA8B,CACrD,GAAI,GAAC,KAAK,cAAgB,CAAC,KAAK,WAKhC,QAFA,KAAK,UAAU,KAAK,EACpB,KAAK,UAAU,UAAU,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,MAAM,EACxE,KAAK,eAAgB,CAC3B,YACA,aACA,YACE,KAAK,iBAAiBA,EAAS,KAAK,UAAgC,EACpE,MACF,UACE,KAAK,UAAUA,CAAO,EACtB,MACF,WACE,KAAK,WAAWA,CAAO,EACvB,KACJ,CACA,KAAK,UAAU,QAAQ,EACvB,KAAK,YAAcA,EACrB,EAeA,KAAQ,iBAAmB,CAACA,EAA2BF,IAAmC,CACxF,GACE,CAAC,KAAK,OACN,CAAC,KAAK,cACN,CAAC,KAAK,WACN,KAAK,yBACL,KAAK,wBAEL,OAEF,KAAK,UAAU,OAAS,OACxB,KAAK,UAAU,sBAAwB,GACvC,KAAK,UAAU,sBAAwB,OAEvC,KAAK,UAAU,yBAA2B,aAC1C,IAAMG,EAAUH,aAAsB,iBAAmBA,EAAW,WAAaA,EAAW,MACtFI,EAAWJ,aAAsB,iBAAmBA,EAAW,YAAcA,EAAW,OAE9F,KAAK,UAAU,UACbA,EACA,EACA,EACAG,EACAC,EACA,EACA,EACA,KAAK,aAAa,MAClB,KAAK,aAAa,MACpB,EACA,KAAK,UAAU,yBAA2B,kBAC1C,KAAK,UAAU,UAAUF,EAAQ,iBAAkB,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,MAAM,EAE1G,KAAK,UAAU,yBAA2B,mBAC1C,KAAK,UAAU,UAAU,KAAK,MAAO,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,MAAM,CAC9F,EA7OE,KAAK,WAAaF,EAClB,KAAK,eAAiBC,EACtB,KAAK,UAAY,KACjB,KAAK,eAAiB,EACtB,KAAK,kBAAoB,KACzB,KAAK,cAAgB,SAAS,cAAc,QAAQ,EACpD,KAAK,eAAiB,KAAK,cAAc,WAAW,IAAI,EAExD,KAAK,cAAc,KAAK,WAAY,KAAK,cAAc,EACvD,KAAK,IAAI,uCAAuC,CAClD,CAEA,aAAuB,CACrB,OAAO,KAAK,aAAa,EAAE,WAC7B,CAEA,iBAA2B,CACzB,MAAO,WAAY,yBAAyB,SAC9C,CAEA,cAAuC,CACrC,IAAMI,EAAgB,CAAC,EACvB,MAAI,CAAC,SAAU,UAAW,MAAO,OAAQ,QAAQ,EAAE,KAAKC,GAAS,UAAU,UAAU,QAAQA,CAAK,IAAM,EAAE,EACxGD,EAAc,YAAc,IAE5BA,EAAc,YAAc,GAC5BA,EAAc,QAAU,4BAA0B,uBAClDA,EAAc,OAAS,8CAGlBA,CACT,CAEA,SAAkB,CAChB,MAAO,OACT,CAEA,eAAoC,CAClC,OAAO,qBAAmB,SAC5B,CAEM,MAAsB,QAAAE,EAAA,sBACrB,KAAK,eACR,KAAK,aAAe,IAAI,qBAAmB,CACzC,WAAaC,GACJ,mEAAmEA,CAAI,EAElF,CAAC,EACD,KAAK,aAAa,WAAW,CAAE,WAAY,GAAO,eAAgB,CAAE,CAAC,EACrE,KAAK,aAAa,UAAU,KAAK,aAAa,EAElD,GASM,cAAcR,EAAkCC,EAA2C,QAAAM,EAAA,sBAC/F,GAAI,CAACP,EACH,MAAM,IAAI,MAAM,8EAA8E,EAGhG,OADA,KAAK,YAAc,OACXC,EAAgB,CACtB,WACA,WACE,KAAK,WAAaD,EAClB,KAAK,eAAiBC,EACtB,MACF,YACE,KAAK,IAAI,8BAA+BD,CAAU,EAElD,IAAMS,EAAM,MAAM,KAAK,SAAST,CAA8B,EAC9D,GAAI,CAACS,GAAO,CAACA,EAAI,UAAY,CAACA,EAAI,cAChC,MAAM,IAAI,MAAM,yEAAyE,EAEzF,KAAK,WAAaA,EAClB,KAAK,uBAEP,MACF,YACE,KAAK,IAAI,8BAA+BT,CAAU,EAClD,KAAK,sBACL,KAAK,WAAaA,EAClB,KAAK,WAAW,YAAc,YAC9B,KAAK,WAAW,MAAQ,GACxB,KAAK,WAAW,KAAO,GACvB,KAAK,WAAW,YAAc,GAC9B,KAAK,WAAW,iBAAmB,IAAYO,EAAA,sBAC7C,GAAI,KAAK,YAAc,KAAK,sBAAsB,iBAChD,GAAI,CACF,MAAM,KAAK,WAAW,KAAK,EAC3B,KAAK,sBACP,OAASG,EAAG,CACV,KAAK,IAAI,4BAA6BV,CAAU,CAClD,CAEJ,GACA,MACF,aACE,KAAK,WAAaA,EAClB,KAAK,wBACL,MACF,UAKE,GAJA,KAAK,IAAI,4BAA6BA,CAAU,EAChD,KAAK,sBACL,KAAK,WAAaA,EAClB,KAAK,UAAY,MAAM,KAAK,QAAQ,KAAK,UAAU,EAC/C,KAAK,WAAa,MAAQ,KAAK,UAAU,OAAS,EACpD,KAAK,yBAEL,OAAM,IAAI,MAAM,8EAA8E,EAEhG,MACF,QACE,KAAK,IACH,6EAA6EW,CAAyB,EACxG,CACJ,CACF,GAEA,eAAgB,CACd,OAAO,KAAK,UACd,CAEA,MAAa,CA3Jf,IAAAC,EA4JQ,KAAK,yBAAqD,KAAK,uBACjEA,EAAA,KAAK,eAAL,MAAAA,EAAmB,SAGrB,KAAK,kBAAoB,KACzB,KAAK,UAAY,KACjB,KAAK,eAAiB,EACtB,KAAK,kBACL,KAAK,qBACP,CAEM,kBAAkBC,EAA0BC,EAA2BC,EAA0B,QAAAR,EAAA,sBAvKzG,IAAAK,EAwKI,GAAI,CAACC,GAAS,CAACC,EACb,MAAM,IAAI,MAAM,6BAA6B,EAO/C,GALA,KAAK,MAAQD,EACbC,EAAO,MAAQD,EAAM,MACrBC,EAAO,OAASD,EAAM,OACtB,KAAK,aAAeC,EACpB,KAAK,UAAYA,EAAO,WAAW,IAAI,EACnCC,GAAkB,KAAK,YAAa,CACtC,KAAK,cAAc,KAAK,WAAW,EACnC,MACF,CACA,GAAI,KAAK,wBAAmD,EAC1DH,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAUC,EAAO,EAAG,EAAGA,EAAM,MAAOA,EAAM,QAC1D,MACF,CACA,MAAM,KAAK,aAAa,KAAK,CAAE,MAAOA,CAAM,CAAC,CAC/C,GAEc,SAASG,EAAuC,QAAAT,EAAA,sBAC5D,OAAAS,EAAM,YAAc,YACb,IAAI,QAAQ,CAACC,EAASC,IAAW,CACtCF,EAAM,OAAS,IAAMC,EAAQD,CAAK,EAClCA,EAAM,QAAUE,CAClB,CAAC,CACH,GAyBQ,QAAQC,EAA2B,CACzC,OAAO,MAAMA,CAAG,EACb,KAAKC,GAAQA,EAAK,YAAY,CAAC,EAC/B,KAAKC,MAAQ,YAASA,CAAI,CAAC,EAC3B,KAAKC,MACG,oBAAiBA,EAAK,EAAI,CAClC,CACL,CAEQ,OAAOC,EAAa,CAC1B,QAAQ,MAAM,KAAK,IAAK,GAAGA,CAAI,CACjC,CAsCQ,WAAWrB,EAA2B,CA3QhD,IAAAU,EAAAY,EAAAC,EA4QQ,CAAC,KAAK,cAAgB,CAAC,KAAK,WAAa,KAAK,0BAGlD,KAAK,UAAW,OAAS,OACzB,KAAK,UAAW,yBAA2B,cAC3Cb,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAUV,EAAQ,MAAO,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,QAC1F,KAAK,UAAW,yBAA2B,oBAC3CsB,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAUtB,EAAQ,iBAAkB,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,QACrG,KAAK,UAAW,OAAS,QAAQ,KAAK,MAAM,KAAK,aAAa,MAAQ,GAAG,EAAI,CAAC,OAC9EuB,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAUvB,EAAQ,MAAO,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,QAC5F,CAEQ,UAAUA,EAA2B,CAC3C,GACE,GAAC,KAAK,cACN,CAAC,KAAK,WACN,CAAC,KAAK,gBACN,KAAK,wBAIP,IAAI,KAAK,mBAAqB,KAAM,CAClC,IAAMwB,EAAO,KAAK,UAAU,KAAK,cAAc,EAAE,KACjD,KAAK,cAAc,MAAQA,EAAK,MAChC,KAAK,cAAc,OAASA,EAAK,OACjC,KAAK,kBAAoB,KAAK,eAAe,gBAAgBA,EAAK,MAAOA,EAAK,MAAM,CACtF,CAEA,KAAK,kBAAkB,KAAK,IAAI,KAAK,UAAU,KAAK,cAAc,EAAE,KAAK,EACzE,KAAK,eAAe,aAAa,KAAK,kBAAmB,EAAG,CAAC,EAC7D,KAAK,gBAAkB,KAAK,eAAiB,GAAK,KAAK,UAAU,OACjE,KAAK,iBAAiBxB,EAAS,KAAK,aAAa,EACnD,CACF,EE7SA,IAAAyB,EAAqB,uBCAd,IAAMC,EAAqB,6CDO3B,IAAMC,EAAN,KAAuD,CAa5D,YAAYC,EAAuBC,EAAqB,CAVxD,KAAQ,WAAa,EACrB,KAAQ,kBACR,KAAQ,sBACR,KAAQ,OAAiC,WACzC,KAAQ,YAAc,GACtB,KAAQ,WAAkC,KAMxC,KAAK,QAAU,IAAI,OAAKD,CAAa,EACrC,KAAK,OAASC,EACd,KAAK,QAAQ,OAAO,CAClB,QAASC,EACT,OAAQ,CACN,eAAgB,GAChB,aAAc,GACd,WAAY,EACd,EACA,UAAW,CACT,gBAAiB,GAAGA,CAAkB,gBACtC,qBAAsB,GAAGA,CAAkB,oBAC7C,CACF,CAAC,EACD,KAAK,OAAS,SAAS,cAAc,QAAQ,EAC7C,KAAK,QAAQ,QAAQC,GAAO,EAEtB,CAACA,EAAI,MAAQA,EAAI,OAAS,UAC5B,QAAQ,MAAM,qBAAsBA,CAAG,CAE3C,CAAC,EACD,KAAK,QAAQ,MAAM,EACnB,KAAK,QAAQ,QAAU,IAAM,CA3CjC,IAAAC,EA4CU,KAAK,UACP,KAAK,YAAc,IACnBA,EAAA,KAAK,SAAL,MAAAA,EAAA,WACA,KAAK,QAAQ,IAAI,EACjB,KAAK,QAAQ,qBAAqB,MAAM,EACxC,KAAK,QAAQ,sBAAsB,KAAK,MAAM,EAC9C,KAAK,YAAY,EAErB,CACF,CAEA,SAAkB,CAChB,MAAO,YACT,CAEQ,iBAAiBC,EAAsB,CACzC,KAAK,aACPA,EAAS,EAGP,KAAK,aAAe,MACtB,cAAc,KAAK,UAAU,EAE/B,KAAK,WAAa,YAAY,IAAM,CAC9B,KAAK,cACP,cAAc,KAAK,UAAW,EAC9BA,EAAS,EAEb,EAAG,GAAG,CACR,CAEA,YAAa,CACX,KAAK,WAAa,EAClB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,UAAU,CACzB,CAAC,CACH,CAEA,kBAAmB,CACjB,KAAK,WAAa,GAClB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,gBAAgB,CAC/B,CAAC,CACH,CAKA,QAAQC,EAAc,CACpB,KAAK,WAAaA,EAClB,KAAK,sBACL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,QAAQ,KAAK,UAAU,CACtC,CAAC,CACH,CAKM,UAAUC,EAAgC,QAAAC,EAAA,sBAC9C,YAAK,OAASD,EACP,IAAI,QAAQ,CAACE,EAASC,IAAW,CACtC,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,sBAAsB,KAAK,MAAM,EAAE,KAAKD,CAAO,EAAE,MAAMC,CAAM,CAC5E,CAAC,CACH,CAAC,CACH,GAEA,mBAAmBL,EAAmD,CACpE,KAAK,2BAA6BA,CACpC,CAEA,WAAY,CACV,OAAO,KAAK,MACd,CAEA,eAAgB,CACd,KAAK,sBACL,KAAK,iBAAiB,EACtB,KAAK,WAAW,CAClB,CAEA,cAAcM,EAA2B,CACvC,KAAK,WAAaA,EAClB,KAAK,uBACL,KAAK,WAAW,EAChB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,cAAc,KAAK,UAAU,CAC5C,CAAC,CACH,CAEA,eAAgB,CACd,OAAO,KAAK,UACd,CAEA,eAAgB,CACd,OAAO,KAAK,YAAc,KAAK,cACjC,CAEQ,aAAaC,EAAqB,CACxC,GAAM,CAAE,OAAAC,EAAQ,MAAAC,CAAM,EAAIF,EAAO,eAAe,EAAE,CAAC,EAAE,YAAY,EACjE,KAAK,OAAO,MAAQE,EACpB,KAAK,OAAO,OAASD,EACrB,KAAK,QAAQ,UAAUD,CAAM,EAC7B,KAAK,QAAQ,SAAS,KAAK,MAAM,CACnC,CAEA,MAAMA,EAAkC,CACtC,YAAK,QAAQ,MAAM,EACnB,KAAK,YAAY,EACjB,KAAK,QAAQ,wBAAwB,IAAM,CA3J/C,IAAAR,EA4JM,KAAK,aAAaQ,CAAM,EACxB,GAAM,CAAE,OAAAC,EAAQ,MAAAC,CAAM,EAAIF,EAAO,eAAe,EAAE,CAAC,EAAE,YAAY,GACjER,EAAA,KAAK,6BAAL,MAAAA,EAAA,UAAkCU,EAAQD,EAC5C,CAAC,EACD,KAAK,aAAaD,CAAM,EACjB,KAAK,OAAO,cAAc,EAAE,GAAKA,CAC1C,CAEA,MAAO,CACL,KAAK,cAAc,EACnB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,KAAK,CACpB,CAAC,CACH,CAEQ,aAAc,CAChB,KAAK,WACP,KAAK,QAAQ,KAAK,UAAU,EACnB,KAAK,YACd,KAAK,cAAc,KAAK,UAAU,CAEtC,CACF",
6
6
  "names": ["require_package", "__commonJSMin", "exports", "module", "src_exports", "__export", "HMSEffectsPlugin", "HMSVBPlugin", "HMSVirtualBackgroundPlugin", "HMSVirtualBackgroundTypes", "__toCommonJS", "import_gifuct_js", "import_hms_video_store", "import_tfjs_backend_webgl", "pkg", "BASE_URL", "TAG", "TFLITE_JS_FILE", "TFLITE_SIMD_JS_FILE", "MODEL_FILE_NAME", "loadScript", "src", "resolve", "reject", "s", "loadTFLiteModel", "__async", "tfLite", "path", "loadTFLite", "modelPath", "modelResponse", "model", "modelBufferOffset", "TAG", "DEFAULT_DELAY", "pkg", "minVideoWidthForSharpening", "maxVideoWidthForSharpening", "minVideoHeightForSharpening", "maxVideoHeightForSharpening", "HMSVirtualBackgroundPlugin", "background", "enableSharpening", "__async", "loadTFLite", "browserResult", "value", "path", "img", "_a", "_b", "input", "output", "skipProcessing", "ctx", "process", "image", "resolve", "reject", "url", "resp", "buff", "gif", "tag", "data", "imageData", "inputMemoryOffset", "outputMemoryOffset", "person", "shift", "backgroundExp", "personExp", "outputImageData", "dims", "dataWidth", "dataHeight", "inputWidth", "inputHeight", "xoffset", "yoffset", "blurValue", "w", "h", "pixels", "weights", "opaque", "side", "halfSide", "src", "sw", "sh", "dst", "alphaFac", "y", "x", "dstOff", "sy", "sx", "r", "g", "b", "a", "cy", "cx", "scy", "scx", "srcOff", "wt", "import_selfie_segmentation", "import_gifuct_js", "import_hms_video_store", "HMSVirtualBackgroundTypes", "HMSVBPlugin", "background", "backgroundType", "results", "bgWidth", "bgHeight", "browserResult", "value", "__async", "file", "img", "e", "HMSVirtualBackgroundTypes", "_a", "input", "output", "skipProcessing", "image", "resolve", "reject", "url", "resp", "buff", "gif", "data", "_b", "_c", "dims", "import_effects_sdk", "EFFECTS_SDK_ASSETS", "HMSEffectsPlugin", "effectsSDKKey", "onInit", "EFFECTS_SDK_ASSETS", "err", "_a", "callback", "blur", "preset", "__async", "resolve", "reject", "url", "stream", "height", "width"]
7
7
  }
package/dist/esm/index.js CHANGED
@@ -1,2 +1,2 @@
1
- var _=(o,t)=>()=>(t||o((t={exports:{}}).exports,t),t.exports);var n=(o,t,e)=>new Promise((i,s)=>{var r=h=>{try{u(e.next(h))}catch(d){s(d)}},a=h=>{try{u(e.throw(h))}catch(d){s(d)}},u=h=>h.done?i(h.value):Promise.resolve(h.value).then(r,a);u((e=e.apply(o,t)).next())});var E=_((gt,j)=>{j.exports={version:"1.13.27",license:"MIT",name:"@100mslive/hms-virtual-background",author:"100ms",module:"dist/esm/index.js",main:"dist/cjs/index.js",typings:"dist/index.d.ts",typesVersions:{"*":{hmsvbplugin:["./dist/HMSVBPlugin.d.ts"],hmseffectsplugin:["./dist/HMSEffectsPlugin.d.ts"]}},exports:{".":{import:"./dist/esm/index.js",require:"./dist/cjs/index.js",default:"./dist/esm/index.js",types:"./dist/index.d.ts"},"./hmsvbplugin":{import:"./dist/esm/HMSVBPlugin.js",require:"./dist/cjs/HMSVBPlugin.js",default:"./dist/esm/HMSVBPlugin.js",types:"./dist/HMSVBPlugin.d.ts"},"./hmseffectsplugin":{import:"./dist/esm/HMSEffectsPlugin.js",default:"./dist/esm/HMSEffectsPlugin.js",require:"./dist/cjs/HMSEffectsPlugin.js",types:"./dist/HMSEffectsPlugin.d.ts"}},repository:{type:"git",url:"https://github.com/100mslive/web-sdks.git",directory:"packages/hms-virtual-background"},files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video-store":"0.12.27"},devDependencies:{"@100mslive/hms-video-store":"0.12.27"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","effects-sdk":"3.4.3","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"bc3bda8727cd8240b6cf3cc6d5ae4e511fdc679b"}});import{decompressFrames as J,parseGIF as K}from"gifuct-js";import{HMSPluginUnsupportedTypes as Q,HMSVideoPluginType as Y}from"@100mslive/hms-video-store";import"@tensorflow/tfjs-backend-webgl";var U=E(),T=`https://unpkg.com/${U.name}/src`,C="VBProcessor",W="tflite/tflite.js",$="tflite/tflite-simd.js",q="models/selfie_segmentation_landscape.tflite",A=o=>new Promise(function(t,e){let i=document.createElement("script");i.src=o,i.onload=t,i.onerror=e,document.head.appendChild(i)}),z=()=>n(void 0,null,function*(){let o,t=`${T}/${$}`;yield A(t);try{o=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${T}/${W}`,yield A(t),o=yield createTFLiteModule()}return o}),D=()=>n(void 0,null,function*(){let o=`${T}/${q}`,[t,e]=yield Promise.all([z(),fetch(o)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var M="VBProcessor",X=33,Z=E(),tt=214,et=855,it=120,st=720,R=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(M,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return n(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(M,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=D(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=Q.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return Z.name}getPluginType(){return Y.TRANSFORM}setBackground(t){return n(this,null,function*(){if(t!=="")if(t==="none")this.log(M,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(M,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>n(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>n(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return n(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>K(e)).then(e=>J(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],a=Math.max(s,r),u=Math.exp(s-a),h=Math.exp(r-a);this.personMask.data[i*4+3]=255*h/(u+h)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>tt&&this.output.height>it&&this.output.width<et&&this.output.height<st&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/X?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return n(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,a,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,a=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,a=(e-s)/2),this.outputCtx.drawImage(t,a,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return n(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),a=t.data,u=t.width,h=t.height,d=u,x=h,B=this.filters.createImageData(d,x),m=B.data,O=i?1:0;for(let g=0;g<x;g=g+1)for(let l=0;l<d;l=l+1){let p=(g*d+l)*4;if(a[p+3]!==0&&l<d&&g<x){let N=g,G=l,V=0,P=0,L=0,I=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let w=N+f-r,S=G+k-r;if(w>=0&&w<h&&S>=0&&S<u){let b=(w*u+S)*4,v=e[f*s+k];V+=a[b]*v,P+=a[b+1]*v,L+=a[b+2]*v,I+=a[b+3]*v}}m[p]=V,m[p+1]=P,m[p+2]=L,m[p+3]=I+O*(255-I)}}return B}}};import{SelfieSegmentation as at}from"@mediapipe/selfie_segmentation";import{decompressFrames as nt,parseGIF as ot}from"gifuct-js";import{HMSPluginUnsupportedTypes as rt,HMSVideoPluginType as ut}from"@100mslive/hms-video-store";var c=(a=>(a.BLUR="blur",a.NONE="none",a.GIF="gif",a.IMAGE="image",a.VIDEO="video",a.CANVAS="canvas",a))(c||{});var F=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.background="none";this.backgroundType="none";this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case"image":case"canvas":case"video":this.renderBackground(t,this.background);break;case"gif":this.renderGIF(t);break;case"blur":this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType==="none"||this.backgroundType==="blur")return;this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-out",this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}isBlurSupported(){return"filter"in CanvasRenderingContext2D.prototype}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge","Safari"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=rt.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return ut.TRANSFORM}init(){return n(this,null,function*(){this.segmentation||(this.segmentation=new at({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return n(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case"none":case"blur":this.background=t,this.backgroundType=e;break;case"image":this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType="image";break;case"video":this.log("setting background to video",t),this.backgroundType="none",this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.playsInline=!0,this.background.oncanplaythrough=()=>n(this,null,function*(){if(this.background&&this.background instanceof HTMLVideoElement)try{yield this.background.play(),this.backgroundType="video"}catch(s){this.log("failed to play background",t)}});break;case"canvas":this.background=t,this.backgroundType="canvas";break;case"gif":if(this.log("setting gif to background",t),this.backgroundType="none",this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif";else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${c}`)}})}getBackground(){return this.background}stop(){var t;this.backgroundType!=="blur"&&this.background!=="none"&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background="none",this.backgroundType="none"}processVideoFrame(t,e,i){return n(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType==="none"){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return n(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>ot(e)).then(e=>nt(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!=="blur"||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!=="gif")){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};import{tsvb as ht}from"effects-sdk";var y="https://assets.100ms.live/effectsdk/3.4.3/";var H=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new ht(t),this.onInit=e,this.effects.config({sdk_url:y,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${y}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${y}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return n(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};export{H as HMSEffectsPlugin,F as HMSVBPlugin,R as HMSVirtualBackgroundPlugin,c as HMSVirtualBackgroundTypes};
1
+ var _=(o,t)=>()=>(t||o((t={exports:{}}).exports,t),t.exports);var n=(o,t,e)=>new Promise((i,s)=>{var r=h=>{try{u(e.next(h))}catch(d){s(d)}},a=h=>{try{u(e.throw(h))}catch(d){s(d)}},u=h=>h.done?i(h.value):Promise.resolve(h.value).then(r,a);u((e=e.apply(o,t)).next())});var E=_((gt,j)=>{j.exports={version:"1.13.28",license:"MIT",name:"@100mslive/hms-virtual-background",author:"100ms",module:"dist/esm/index.js",main:"dist/cjs/index.js",typings:"dist/index.d.ts",typesVersions:{"*":{hmsvbplugin:["./dist/HMSVBPlugin.d.ts"],hmseffectsplugin:["./dist/HMSEffectsPlugin.d.ts"]}},exports:{".":{import:"./dist/esm/index.js",require:"./dist/cjs/index.js",default:"./dist/esm/index.js",types:"./dist/index.d.ts"},"./hmsvbplugin":{import:"./dist/esm/HMSVBPlugin.js",require:"./dist/cjs/HMSVBPlugin.js",default:"./dist/esm/HMSVBPlugin.js",types:"./dist/HMSVBPlugin.d.ts"},"./hmseffectsplugin":{import:"./dist/esm/HMSEffectsPlugin.js",default:"./dist/esm/HMSEffectsPlugin.js",require:"./dist/cjs/HMSEffectsPlugin.js",types:"./dist/HMSEffectsPlugin.d.ts"}},repository:{type:"git",url:"https://github.com/100mslive/web-sdks.git",directory:"packages/hms-virtual-background"},files:["dist","src/tflite","src/models"],scripts:{start:'concurrently "yarn dev" "yarn types"',dev:"node ../../scripts/dev","build:only":"node ../../scripts/build",build:"yarn build:only && yarn types:build",types:"tsc -w","types:build":"tsc -p tsconfig.json",test:"jest --maxWorkers=1 --passWithNoTests",lint:"eslint -c ../../.eslintrc .","lint:fix":"yarn lint --fix",prepare:"yarn build",size:"size-limit",analyze:"size-limit --why",format:"prettier --write src/**/*.ts"},peerDependencies:{"@100mslive/hms-video-store":"0.12.28"},devDependencies:{"@100mslive/hms-video-store":"0.12.28"},dependencies:{"@mediapipe/selfie_segmentation":"^0.1.1632777926","@tensorflow-models/body-segmentation":"^1.0.1","@tensorflow/tfjs-backend-webgl":"^3.3.0","@tensorflow/tfjs-converter":"^3.19.0","@tensorflow/tfjs-core":"^3.19.0","@webassemblyjs/helper-wasm-bytecode":"1.11.1","@webassemblyjs/wasm-gen":"1.11.1","effects-sdk":"3.4.3","gifuct-js":"^2.1.2","wasm-check":"^2.0.2"},eslintIgnore:["tflite.js","tflite-simd.js","tflite.wasm","tflite-simd.wasm","defineTFLite.ts","importing.test.ts"],gitHead:"c4d7367340117e66de386fc7aaa9b5e95abd19ed"}});import{decompressFrames as J,parseGIF as K}from"gifuct-js";import{HMSPluginUnsupportedTypes as Q,HMSVideoPluginType as Y}from"@100mslive/hms-video-store";import"@tensorflow/tfjs-backend-webgl";var U=E(),T=`https://unpkg.com/${U.name}/src`,C="VBProcessor",W="tflite/tflite.js",$="tflite/tflite-simd.js",q="models/selfie_segmentation_landscape.tflite",A=o=>new Promise(function(t,e){let i=document.createElement("script");i.src=o,i.onload=t,i.onerror=e,document.head.appendChild(i)}),z=()=>n(void 0,null,function*(){let o,t=`${T}/${$}`;yield A(t);try{o=yield createTFLiteSIMDModule()}catch(e){console.warn("SIMD not supported. You may experience poor virtual background effect."),t=`${T}/${W}`,yield A(t),o=yield createTFLiteModule()}return o}),D=()=>n(void 0,null,function*(){let o=`${T}/${q}`,[t,e]=yield Promise.all([z(),fetch(o)]),i=yield e.arrayBuffer(),s=t._getModelBufferMemoryOffset();return t.HEAPU8.set(new Uint8Array(i),s),t._loadModel(i.byteLength),console.debug(C,"Input memory offset:",t._getInputMemoryOffset()),console.debug(C,"Input height:",t._getInputHeight()),console.debug(C,"Input width:",t._getInputWidth()),console.debug(C,"Input channels:",t._getInputChannelCount()),t});var M="VBProcessor",X=33,Z=E(),tt=214,et=855,it=120,st=720,R=class{constructor(t,e=!1){this.backgroundType="none";this.background=t,this.enableSharpening=e,this.backgroundImage=null,this.backgroundVideo=null,this.personMaskWidth=256,this.personMaskHeight=144,this.isVirtualBackground=!1,this.blurValue="10px",this.loadModelCalled=!1,this.tfLite=null,this.modelName="landscape-segmentation",this.outputCtx=null,this.input=null,this.output=null,this.timerID=0,this.imageAspectRatio=1,this.personMaskPixelCount=this.personMaskWidth*this.personMaskHeight,this.personMask=new ImageData(this.personMaskWidth,this.personMaskHeight),this.personMaskCanvas=document.createElement("canvas"),this.personMaskCanvas.width=this.personMaskWidth,this.personMaskCanvas.height=this.personMaskHeight,this.personMaskCtx=this.personMaskCanvas.getContext("2d"),this.filters={},this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.giflocalCount=0,this.enableSharpening=e,this.log(M,"Virtual Background plugin created"),this.setBackground(this.background)}init(){return n(this,null,function*(){this.loadModelCalled?yield this.tfLitePromise:(this.log(M,"PREVIOUS LOADED MODEL IS ",this.tfLite),this.loadModelCalled=!0,this.tfLitePromise=D(),this.tfLite=yield this.tfLitePromise),this.enableSharpening&&this.initSharpenFilter()})}isSupported(){return navigator.userAgent.indexOf("Chrome")!==-1||navigator.userAgent.indexOf("Firefox")!==-1||navigator.userAgent.indexOf("Edg")!==-1||navigator.userAgent.indexOf("Edge")!==-1}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=Q.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return Z.name}getPluginType(){return Y.TRANSFORM}setBackground(t){return n(this,null,function*(){if(t!=="")if(t==="none")this.log(M,"setting background to :",t),this.background="none",this.backgroundType="none",this.isVirtualBackground=!1;else if(t==="blur")this.log(M,"setting background to :",t),this.background="blur",this.backgroundType="blur",this.isVirtualBackground=!1;else if(t instanceof HTMLImageElement){this.log("setting background to image",t);let e=yield this.setImage(t);if(!e||!e.complete||!e.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.isVirtualBackground=!0,this.backgroundImage=e,this.backgroundType="image"}else if(t instanceof HTMLVideoElement)this.log("setting background to video",t),this.backgroundVideo=t,this.backgroundVideo.crossOrigin="anonymous",this.backgroundVideo.muted=!0,this.backgroundVideo.loop=!0,this.backgroundVideo.oncanplaythrough=()=>n(this,null,function*(){this.backgroundVideo!=null&&(yield this.backgroundVideo.play(),this.isVirtualBackground=!0,this.backgroundType="video")});else if(console.log("setting gif to background"),this.gifFrames=yield this.setGiF(t),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif",this.isVirtualBackground=!0;else throw new Error("Invalid background supplied, see the docs to check supported background type");else throw new Error("Invalid background supplied, see the docs to check supported background type")})}stop(){var t,e;this.isVirtualBackground&&((t=this.backgroundImage)==null||t.removeAttribute("src"),(e=this.backgroundVideo)==null||e.removeAttribute("src"),this.backgroundType==="video"&&(this.backgroundVideo.loop=!1,this.backgroundVideo=null)),this.outputCtx&&(this.outputCtx.fillStyle="rgb(0, 0, 0)",this.outputCtx.fillRect(0,0,this.output.width,this.output.height)),this.gifFrameImageData=null,this.gifFrames=null,this.giflocalCount=0,this.gifFramesIndex=0}processVideoFrame(t,e,i){if(!t||!e)throw new Error("Plugin invalid input/output");this.input=t,this.output=e;let s=e.getContext("2d");if(s.canvas.width!==t.width&&(s.canvas.width=t.width),s.canvas.height!==t.height&&(s.canvas.height=t.height),this.backgroundType==="video"&&(this.backgroundVideo.width=t.width,this.backgroundVideo.height=t.height),this.outputCtx=s,this.imageAspectRatio=t.width/t.height,this.imageAspectRatio<=0)throw new Error("Invalid input width/height");let r=()=>n(this,null,function*(){yield this.runSegmentation(i)});this.background==="none"&&!this.isVirtualBackground?(this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.outputCtx.drawImage(t,0,0,t.width,t.height)):r()}setImage(t){return n(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}setGiF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>K(e)).then(e=>J(e,!0))}log(t,...e){console.info(t,...e)}resizeInputData(){this.personMaskCtx.drawImage(this.input,0,0,this.input.width,this.input.height,0,0,this.personMaskWidth,this.personMaskHeight);let t=this.personMaskCtx.getImageData(0,0,this.personMaskWidth,this.personMaskHeight),e=this.tfLite._getInputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)this.tfLite.HEAPF32[e+i*3]=t.data[i*4]/255,this.tfLite.HEAPF32[e+i*3+1]=t.data[i*4+1]/255,this.tfLite.HEAPF32[e+i*3+2]=t.data[i*4+2]/255}infer(t){t||this.tfLite._runInference();let e=this.tfLite._getOutputMemoryOffset()/4;for(let i=0;i<this.personMaskPixelCount;i++)if(this.modelName==="meet"){let s=this.tfLite.HEAPF32[e+i*2],r=this.tfLite.HEAPF32[e+i*2+1],a=Math.max(s,r),u=Math.exp(s-a),h=Math.exp(r-a);this.personMask.data[i*4+3]=255*h/(u+h)}else if(this.modelName==="landscape-segmentation"){let s=this.tfLite.HEAPF32[e+i];this.personMask.data[i*4+3]=255*s}this.personMaskCtx.putImageData(this.personMask,0,0)}postProcessing(){this.outputCtx.globalCompositeOperation="copy",this.outputCtx.filter="none",this.isVirtualBackground?this.outputCtx.filter="blur(4px)":this.outputCtx.filter="blur(8px)",this.drawPersonMask(),this.outputCtx.globalCompositeOperation="source-in",this.outputCtx.filter="none",this.outputCtx.drawImage(this.input,0,0),this.enableSharpening&&this.output.width>tt&&this.output.height>it&&this.output.width<et&&this.output.height<st&&this.sharpenFilter(),this.drawSegmentedBackground()}sharpenFilter(){let t=this.outputCtx.getImageData(0,0,this.output.width,this.output.height),e=this.filters.convolute(t);this.outputCtx.putImageData(e,0,0)}drawPersonMask(){this.outputCtx.drawImage(this.personMaskCanvas,0,0,this.personMaskWidth,this.personMaskHeight,0,0,this.output.width,this.output.height)}drawSegmentedBackground(){this.outputCtx.globalCompositeOperation="destination-over",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.isVirtualBackground?this.backgroundType==="video"&&this.backgroundVideo!=null&&this.backgroundVideo.readyState>=4?this.fitVideoToBackground():this.backgroundType==="image"?this.fitImageToBackground():this.backgroundType==="gif"&&(this.giflocalCount>this.gifFrames[this.gifFramesIndex].delay/X?(this.gifFramesIndex++,this.gifFramesIndex>=this.gifFrames.length&&(this.gifFramesIndex=0),this.giflocalCount=0):this.giflocalCount++,this.fitGifToBackground()):this.addBlurToBackground()}runSegmentation(t){return n(this,null,function*(){this.tfLite&&(this.resizeInputData(),yield this.infer(t),this.postProcessing())})}fitVideoToBackground(){this.fitData(this.backgroundVideo,this.backgroundVideo.videoWidth,this.backgroundVideo.videoHeight)}fitImageToBackground(){this.fitData(this.backgroundImage,this.backgroundImage.width,this.backgroundImage.height)}fitGifToBackground(){if(this.gifFrameImageData==null){let t=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=t.width,this.tempGifCanvas.height=t.height,this.gifFrameImageData=this.tempGifContext.createImageData(t.width,t.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.fitData(this.tempGifCanvas,this.gifFrameImageData.width,this.gifFrameImageData.height)}fitData(t,e,i){let s,r,a,u;e/i<this.imageAspectRatio?(s=e,r=e/this.imageAspectRatio,a=0,u=(i-r)/2):(r=i,s=i*this.imageAspectRatio,u=0,a=(e-s)/2),this.outputCtx.drawImage(t,a,u,s,r,0,0,this.output.width,this.output.height)}addBlurToBackground(){return n(this,null,function*(){let t="15px";this.input.width<=160?t="5px":this.input.width<=320?t="10px":this.input.width<=640?t="15px":this.input.width<=960?t="20px":this.input.width<=1280?t="25px":this.input.width<=1920&&(t="30px"),this.outputCtx.filter=`blur(${t})`,this.outputCtx.drawImage(this.input,0,0,this.output.width,this.output.height)})}initSharpenFilter(){this.filters.tmpCanvas=document.createElement("canvas"),this.filters.tmpCtx=this.filters.tmpCanvas.getContext("2d"),this.filters.createImageData=(t,e)=>this.filters.tmpCtx.createImageData(t,e),this.filters.convolute=(t,e=[0,-1,0,-1,5,-1,0,-1,0],i)=>{let s=Math.round(Math.sqrt(e.length)),r=Math.floor(s/2),a=t.data,u=t.width,h=t.height,d=u,x=h,B=this.filters.createImageData(d,x),m=B.data,O=i?1:0;for(let g=0;g<x;g=g+1)for(let l=0;l<d;l=l+1){let p=(g*d+l)*4;if(a[p+3]!==0&&l<d&&g<x){let N=g,G=l,V=0,P=0,L=0,I=0;for(let f=0;f<s;f++)for(let k=0;k<s;k++){let w=N+f-r,S=G+k-r;if(w>=0&&w<h&&S>=0&&S<u){let b=(w*u+S)*4,v=e[f*s+k];V+=a[b]*v,P+=a[b+1]*v,L+=a[b+2]*v,I+=a[b+3]*v}}m[p]=V,m[p+1]=P,m[p+2]=L,m[p+3]=I+O*(255-I)}}return B}}};import{SelfieSegmentation as at}from"@mediapipe/selfie_segmentation";import{decompressFrames as nt,parseGIF as ot}from"gifuct-js";import{HMSPluginUnsupportedTypes as rt,HMSVideoPluginType as ut}from"@100mslive/hms-video-store";var c=(a=>(a.BLUR="blur",a.NONE="none",a.GIF="gif",a.IMAGE="image",a.VIDEO="video",a.CANVAS="canvas",a))(c||{});var F=class{constructor(t,e){this.TAG="[HMSVBPlugin]";this.background="none";this.backgroundType="none";this.handleResults=t=>{if(!(!this.outputCanvas||!this.outputCtx)){switch(this.outputCtx.save(),this.outputCtx.clearRect(0,0,this.outputCanvas.width,this.outputCanvas.height),this.backgroundType){case"image":case"canvas":case"video":this.renderBackground(t,this.background);break;case"gif":this.renderGIF(t);break;case"blur":this.renderBlur(t);break}this.outputCtx.restore(),this.prevResults=t}};this.renderBackground=(t,e)=>{if(!this.input||!this.outputCanvas||!this.outputCtx||this.backgroundType==="none"||this.backgroundType==="blur")return;this.outputCtx.filter="none",this.outputCtx.imageSmoothingEnabled=!0,this.outputCtx.imageSmoothingQuality="high",this.outputCtx.globalCompositeOperation="source-out";let i=e instanceof HTMLVideoElement?e.videoWidth:e.width,s=e instanceof HTMLVideoElement?e.videoHeight:e.height;this.outputCtx.drawImage(e,0,0,i,s,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-out",this.outputCtx.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",this.outputCtx.drawImage(this.input,0,0,this.outputCanvas.width,this.outputCanvas.height)};this.background=t,this.backgroundType=e,this.gifFrames=null,this.gifFramesIndex=0,this.gifFrameImageData=null,this.tempGifCanvas=document.createElement("canvas"),this.tempGifContext=this.tempGifCanvas.getContext("2d"),this.setBackground(this.background,this.backgroundType),this.log("Virtual background plugin initialised")}isSupported(){return this.checkSupport().isSupported}isBlurSupported(){return"filter"in CanvasRenderingContext2D.prototype}checkSupport(){let t={};return["Chrome","Firefox","Edg","Edge","Safari"].some(e=>navigator.userAgent.indexOf(e)!==-1)?t.isSupported=!0:(t.isSupported=!1,t.errType=rt.PLATFORM_NOT_SUPPORTED,t.errMsg="browser not supported for plugin, see docs"),t}getName(){return"HMSVB"}getPluginType(){return ut.TRANSFORM}init(){return n(this,null,function*(){this.segmentation||(this.segmentation=new at({locateFile:t=>`https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${t}`}),this.segmentation.setOptions({selfieMode:!1,modelSelection:1}),this.segmentation.onResults(this.handleResults))})}setBackground(t,e){return n(this,null,function*(){if(!t)throw new Error("Invalid background supplied, see the docs to check supported background type");switch(this.prevResults=void 0,e){case"none":case"blur":this.background=t,this.backgroundType=e;break;case"image":this.log("setting background to image",t);let i=yield this.setImage(t);if(!i||!i.complete||!i.naturalHeight)throw new Error("Invalid image. Provide a valid and successfully loaded HTMLImageElement");this.background=i,this.backgroundType="image";break;case"video":this.log("setting background to video",t),this.backgroundType="none",this.background=t,this.background.crossOrigin="anonymous",this.background.muted=!0,this.background.loop=!0,this.background.playsInline=!0,this.background.oncanplaythrough=()=>n(this,null,function*(){if(this.background&&this.background instanceof HTMLVideoElement)try{yield this.background.play(),this.backgroundType="video"}catch(s){this.log("failed to play background",t)}});break;case"canvas":this.background=t,this.backgroundType="canvas";break;case"gif":if(this.log("setting gif to background",t),this.backgroundType="none",this.background=t,this.gifFrames=yield this.loadGIF(this.background),this.gifFrames!=null&&this.gifFrames.length>0)this.backgroundType="gif";else throw new Error("Invalid background supplied, see the docs to check supported background type");break;default:this.log(`backgroundType did not match with any of the supported background types - ${c}`)}})}getBackground(){return this.background}stop(){var t;this.backgroundType!=="blur"&&this.background!=="none"&&((t=this.segmentation)==null||t.reset()),this.gifFrameImageData=null,this.gifFrames=null,this.gifFramesIndex=0,this.background="none",this.backgroundType="none"}processVideoFrame(t,e,i){return n(this,null,function*(){var s;if(!t||!e)throw new Error("Plugin invalid input/output");if(this.input=t,e.width=t.width,e.height=t.height,this.outputCanvas=e,this.outputCtx=e.getContext("2d"),i&&this.prevResults){this.handleResults(this.prevResults);return}if(this.backgroundType==="none"){(s=this.outputCtx)==null||s.drawImage(t,0,0,t.width,t.height);return}yield this.segmentation.send({image:t})})}setImage(t){return n(this,null,function*(){return t.crossOrigin="anonymous",new Promise((e,i)=>{t.onload=()=>e(t),t.onerror=i})})}loadGIF(t){return fetch(t).then(e=>e.arrayBuffer()).then(e=>ot(e)).then(e=>nt(e,!0))}log(...t){console.debug(this.TAG,...t)}renderBlur(t){var e,i,s;!this.outputCanvas||!this.outputCtx||this.backgroundType!=="blur"||(this.outputCtx.filter="none",this.outputCtx.globalCompositeOperation="source-out",(e=this.outputCtx)==null||e.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.globalCompositeOperation="destination-atop",(i=this.outputCtx)==null||i.drawImage(t.segmentationMask,0,0,this.outputCanvas.width,this.outputCanvas.height),this.outputCtx.filter=`blur(${Math.floor(this.outputCanvas.width/160)*5}px)`,(s=this.outputCtx)==null||s.drawImage(t.image,0,0,this.outputCanvas.width,this.outputCanvas.height))}renderGIF(t){if(!(!this.outputCanvas||!this.outputCtx||!this.tempGifContext||this.backgroundType!=="gif")){if(this.gifFrameImageData==null){let e=this.gifFrames[this.gifFramesIndex].dims;this.tempGifCanvas.width=e.width,this.tempGifCanvas.height=e.height,this.gifFrameImageData=this.tempGifContext.createImageData(e.width,e.height)}this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch),this.tempGifContext.putImageData(this.gifFrameImageData,0,0),this.gifFramesIndex=(this.gifFramesIndex+1)%this.gifFrames.length,this.renderBackground(t,this.tempGifCanvas)}}};import{tsvb as ht}from"effects-sdk";var y="https://assets.100ms.live/effectsdk/3.4.3/";var H=class{constructor(t,e){this.blurAmount=0;this.background="none";this.backgroundType="none";this.preset="balanced";this.initialised=!1;this.intervalId=null;this.effects=new ht(t),this.onInit=e,this.effects.config({sdk_url:y,models:{colorcorrector:"",facedetector:"",lowlighter:""},wasmPaths:{"ort-wasm.wasm":`${y}ort-wasm.wasm`,"ort-wasm-simd.wasm":`${y}ort-wasm-simd.wasm`}}),this.canvas=document.createElement("canvas"),this.effects.onError(i=>{(!i.type||i.type==="error")&&console.error("[HMSEffectsPlugin]",i)}),this.effects.cache(),this.effects.onReady=()=>{var i;this.effects&&(this.initialised=!0,(i=this.onInit)==null||i.call(this),this.effects.run(),this.effects.setBackgroundFitMode("fill"),this.effects.setSegmentationPreset(this.preset),this.applyEffect())}}getName(){return"HMSEffects"}executeAfterInit(t){this.initialised&&t(),this.intervalId!==null&&clearInterval(this.intervalId),this.intervalId=setInterval(()=>{this.initialised&&(clearInterval(this.intervalId),t())},100)}removeBlur(){this.blurAmount=0,this.executeAfterInit(()=>{this.effects.clearBlur()})}removeBackground(){this.background="",this.executeAfterInit(()=>{this.effects.clearBackground()})}setBlur(t){this.blurAmount=t,this.backgroundType="blur",this.removeBackground(),this.executeAfterInit(()=>{this.effects.setBlur(this.blurAmount)})}setPreset(t){return n(this,null,function*(){return this.preset=t,new Promise((e,i)=>{this.executeAfterInit(()=>{this.effects.setSegmentationPreset(this.preset).then(e).catch(i)})})})}onResolutionChange(t){this.onResolutionChangeCallback=t}getPreset(){return this.preset}removeEffects(){this.backgroundType="none",this.removeBackground(),this.removeBlur()}setBackground(t){this.background=t,this.backgroundType="image",this.removeBlur(),this.executeAfterInit(()=>{this.effects.setBackground(this.background)})}getBlurAmount(){return this.blurAmount}getBackground(){return this.background||this.backgroundType}updateCanvas(t){let{height:e,width:i}=t.getVideoTracks()[0].getSettings();this.canvas.width=i,this.canvas.height=e,this.effects.useStream(t),this.effects.toCanvas(this.canvas)}apply(t){return this.effects.clear(),this.applyEffect(),this.effects.onChangeInputResolution(()=>{var s;this.updateCanvas(t);let{height:e,width:i}=t.getVideoTracks()[0].getSettings();(s=this.onResolutionChangeCallback)==null||s.call(this,i,e)}),this.updateCanvas(t),this.canvas.captureStream(30)||t}stop(){this.removeEffects(),this.executeAfterInit(()=>{this.effects.stop()})}applyEffect(){this.blurAmount?this.setBlur(this.blurAmount):this.background&&this.setBackground(this.background)}};export{H as HMSEffectsPlugin,F as HMSVBPlugin,R as HMSVirtualBackgroundPlugin,c as HMSVirtualBackgroundTypes};
2
2
  //# sourceMappingURL=index.js.map
package/dist/esm/index.js.map CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../package.json", "../../src/HMSVirtualBackgroundPlugin.ts", "../../src/defineTFLite.ts", "../../src/HMSVBPlugin.ts", "../../src/interfaces.ts", "../../src/HMSEffectsPlugin.ts", "../../src/constants.ts"],
4
- "sourcesContent": ["{\n \"version\": \"1.13.27\",\n \"license\": \"MIT\",\n \"name\": \"@100mslive/hms-virtual-background\",\n \"author\": \"100ms\",\n \"module\": \"dist/esm/index.js\",\n \"main\": \"dist/cjs/index.js\",\n \"typings\": \"dist/index.d.ts\",\n \"typesVersions\": {\n \"*\": {\n \"hmsvbplugin\": [\n \"./dist/HMSVBPlugin.d.ts\"\n ],\n \"hmseffectsplugin\": [\n \"./dist/HMSEffectsPlugin.d.ts\"\n ]\n }\n },\n \"exports\": {\n \".\": {\n \"import\": \"./dist/esm/index.js\",\n \"require\": \"./dist/cjs/index.js\",\n \"default\": \"./dist/esm/index.js\",\n \"types\": \"./dist/index.d.ts\"\n },\n \"./hmsvbplugin\": {\n \"import\": \"./dist/esm/HMSVBPlugin.js\",\n \"require\": \"./dist/cjs/HMSVBPlugin.js\",\n \"default\": \"./dist/esm/HMSVBPlugin.js\",\n \"types\": \"./dist/HMSVBPlugin.d.ts\"\n },\n \"./hmseffectsplugin\": {\n \"import\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"default\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"require\": \"./dist/cjs/HMSEffectsPlugin.js\",\n \"types\": \"./dist/HMSEffectsPlugin.d.ts\"\n }\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"https://github.com/100mslive/web-sdks.git\",\n \"directory\": \"packages/hms-virtual-background\"\n },\n \"files\": [\n \"dist\",\n \"src/tflite\",\n \"src/models\"\n ],\n \"scripts\": {\n \"start\": \"concurrently \\\"yarn dev\\\" \\\"yarn types\\\"\",\n \"dev\": \"node ../../scripts/dev\",\n \"build:only\": \"node ../../scripts/build\",\n \"build\": \"yarn build:only && yarn types:build\",\n \"types\": \"tsc -w\",\n \"types:build\": \"tsc -p tsconfig.json\",\n \"test\": \"jest --maxWorkers=1 --passWithNoTests\",\n \"lint\": \"eslint -c ../../.eslintrc .\",\n \"lint:fix\": \"yarn lint --fix\",\n \"prepare\": \"yarn build\",\n \"size\": \"size-limit\",\n \"analyze\": \"size-limit --why\",\n \"format\": \"prettier --write src/**/*.ts\"\n },\n \"peerDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.27\"\n },\n \"devDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.27\"\n },\n \"dependencies\": {\n \"@mediapipe/selfie_segmentation\": \"^0.1.1632777926\",\n \"@tensorflow-models/body-segmentation\": \"^1.0.1\",\n \"@tensorflow/tfjs-backend-webgl\": \"^3.3.0\",\n \"@tensorflow/tfjs-converter\": \"^3.19.0\",\n \"@tensorflow/tfjs-core\": \"^3.19.0\",\n \"@webassemblyjs/helper-wasm-bytecode\": \"1.11.1\",\n \"@webassemblyjs/wasm-gen\": \"1.11.1\",\n \"effects-sdk\": \"3.4.3\",\n \"gifuct-js\": \"^2.1.2\",\n \"wasm-check\": \"^2.0.2\"\n },\n \"eslintIgnore\": [\n \"tflite.js\",\n \"tflite-simd.js\",\n \"tflite.wasm\",\n \"tflite-simd.wasm\",\n \"defineTFLite.ts\",\n \"importing.test.ts\"\n ],\n \"gitHead\": \"bc3bda8727cd8240b6cf3cc6d5ae4e511fdc679b\"\n}\n", "/* eslint-disable complexity */\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport '@tensorflow/tfjs-backend-webgl';\nimport { loadTFLite } from './defineTFLite';\n\nconst TAG = 'VBProcessor';\nconst DEFAULT_DELAY = 33;\nconst pkg = require('../package.json');\n\nconst minVideoWidthForSharpening = 214;\nconst maxVideoWidthForSharpening = 855;\nconst minVideoHeightForSharpening = 120;\nconst maxVideoHeightForSharpening = 720;\n\nexport class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {\n background: string | HTMLImageElement;\n personMaskWidth: number;\n personMaskHeight: number;\n isVirtualBackground: boolean;\n backgroundImage: HTMLImageElement | null;\n backgroundVideo: HTMLVideoElement | null;\n backgroundType = 'none';\n loadModelCalled: boolean;\n blurValue: any;\n tfLite: any;\n tfLitePromise: any;\n modelName: string;\n\n input: HTMLCanvasElement | null;\n output: HTMLCanvasElement | null;\n outputCtx: CanvasRenderingContext2D | null;\n timerID: number;\n imageAspectRatio: number;\n\n personMaskPixelCount: number;\n personMask: ImageData;\n personMaskCanvas: HTMLCanvasElement;\n personMaskCtx: any;\n filters: any;\n enableSharpening?: boolean | false;\n\n gifFrames: any;\n gifFramesIndex: number;\n gifFrameImageData: any;\n tempGifCanvas: HTMLCanvasElement;\n tempGifContext: any;\n giflocalCount: number;\n\n constructor(background: string, enableSharpening = false) {\n this.background = background;\n this.enableSharpening = enableSharpening;\n\n this.backgroundImage = null;\n this.backgroundVideo = null;\n\n this.personMaskWidth = 256;\n this.personMaskHeight = 144;\n this.isVirtualBackground = false;\n this.blurValue = '10px';\n this.loadModelCalled = false;\n this.tfLite = null;\n this.modelName = 'landscape-segmentation';\n\n this.outputCtx = null;\n this.input = null;\n this.output = null;\n this.timerID = 0;\n this.imageAspectRatio = 1;\n\n this.personMaskPixelCount = this.personMaskWidth * this.personMaskHeight;\n this.personMask = new ImageData(this.personMaskWidth, this.personMaskHeight);\n this.personMaskCanvas = document.createElement('canvas');\n this.personMaskCanvas.width = this.personMaskWidth;\n this.personMaskCanvas.height = this.personMaskHeight;\n this.personMaskCtx = this.personMaskCanvas.getContext('2d');\n\n this.filters = {};\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n this.giflocalCount = 0;\n this.enableSharpening = enableSharpening;\n\n this.log(TAG, 'Virtual Background plugin created');\n this.setBackground(this.background);\n }\n\n async init(): Promise<void> {\n if (!this.loadModelCalled) {\n this.log(TAG, 'PREVIOUS LOADED MODEL IS ', this.tfLite);\n this.loadModelCalled = true;\n this.tfLitePromise = loadTFLite();\n this.tfLite = await this.tfLitePromise;\n } else {\n //either it is loading or loaded\n await this.tfLitePromise;\n }\n if (this.enableSharpening) {\n this.initSharpenFilter();\n }\n }\n\n /*\n @depreceated\n */\n isSupported(): boolean {\n //support chrome, firefox, edge TODO: check this\n return (\n navigator.userAgent.indexOf('Chrome') !== -1 ||\n navigator.userAgent.indexOf('Firefox') !== -1 ||\n navigator.userAgent.indexOf('Edg') !== -1 ||\n navigator.userAgent.indexOf('Edge') !== -1\n );\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return pkg.name;\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async setBackground(path?: string | HTMLImageElement | HTMLVideoElement) {\n if (path !== '') {\n if (path === 'none') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'none';\n this.backgroundType = 'none';\n this.isVirtualBackground = false;\n } else if (path === 'blur') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'blur';\n this.backgroundType = 'blur';\n this.isVirtualBackground = false;\n } else if (path instanceof HTMLImageElement) {\n this.log('setting background to image', path);\n const img = await this.setImage(path as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.isVirtualBackground = true;\n this.backgroundImage = img;\n this.backgroundType = 'image';\n }\n } else if (path instanceof HTMLVideoElement) {\n this.log('setting background to video', path);\n this.backgroundVideo = path as HTMLVideoElement;\n this.backgroundVideo.crossOrigin = 'anonymous';\n this.backgroundVideo.muted = true;\n this.backgroundVideo.loop = true;\n this.backgroundVideo.oncanplaythrough = async () => {\n if (this.backgroundVideo != null) {\n await this.backgroundVideo!.play();\n this.isVirtualBackground = true;\n this.backgroundType = 'video';\n }\n };\n } else {\n console.log('setting gif to background');\n this.gifFrames = await this.setGiF(path as string);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = 'gif';\n this.isVirtualBackground = true;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n\n stop(): void {\n if (this.isVirtualBackground) {\n this.backgroundImage?.removeAttribute('src');\n this.backgroundVideo?.removeAttribute('src');\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.loop = false;\n this.backgroundVideo = null;\n }\n }\n if (this.outputCtx) {\n this.outputCtx!.fillStyle = `rgb(0, 0, 0)`;\n this.outputCtx!.fillRect(0, 0, this.output!.width, this.output!.height);\n }\n\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.giflocalCount = 0;\n this.gifFramesIndex = 0;\n }\n\n processVideoFrame(\n input: HTMLCanvasElement,\n output: HTMLCanvasElement,\n skipProcessing?: boolean,\n ): Promise<void> | void {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n\n this.input = input;\n this.output = output;\n\n const ctx = output.getContext('2d');\n if (ctx!.canvas.width !== input.width) {\n ctx!.canvas.width = input.width;\n }\n if (ctx!.canvas.height !== input.height) {\n ctx!.canvas.height = input.height;\n }\n\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.width = input.width;\n this.backgroundVideo!.height = input.height;\n }\n\n this.outputCtx = ctx!;\n this.imageAspectRatio = input.width / input.height;\n if (this.imageAspectRatio <= 0) {\n throw new Error('Invalid input width/height');\n }\n\n const process = async () => {\n await this.runSegmentation(skipProcessing);\n };\n\n if (this.background === 'none' && !this.isVirtualBackground) {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n this.outputCtx!.drawImage(input, 0, 0, input.width, input.height);\n } else {\n process();\n }\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private setGiF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(tag: string, ...data: any[]) {\n console.info(tag, ...data);\n }\n\n private resizeInputData() {\n this.personMaskCtx!.drawImage(\n this.input,\n 0,\n 0,\n this.input!.width,\n this.input!.height,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n );\n\n const imageData = this.personMaskCtx!.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight);\n //\n const inputMemoryOffset = this.tfLite._getInputMemoryOffset() / 4;\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3] = imageData.data[i * 4] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 1] = imageData.data[i * 4 + 1] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 2] = imageData.data[i * 4 + 2] / 255;\n }\n }\n private infer(skipProcessing?: boolean) {\n if (!skipProcessing) {\n this.tfLite._runInference();\n }\n const outputMemoryOffset = this.tfLite._getOutputMemoryOffset() / 4;\n\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n if (this.modelName === 'meet') {\n const background = this.tfLite.HEAPF32[outputMemoryOffset + i * 2];\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i * 2 + 1];\n const shift = Math.max(background, person);\n const backgroundExp = Math.exp(background - shift);\n const personExp = Math.exp(person - shift);\n // Sets only the alpha component of each pixel.\n this.personMask.data[i * 4 + 3] = (255 * personExp) / (backgroundExp + personExp);\n } else if (this.modelName === 'landscape-segmentation') {\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i];\n this.personMask.data[i * 4 + 3] = 255 * person;\n }\n }\n\n this.personMaskCtx!.putImageData(this.personMask, 0, 0);\n }\n\n private postProcessing() {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n\n if (this.isVirtualBackground) {\n this.outputCtx!.filter = 'blur(4px)';\n } else {\n this.outputCtx!.filter = 'blur(8px)';\n }\n this.drawPersonMask();\n this.outputCtx!.globalCompositeOperation = 'source-in';\n this.outputCtx!.filter = 'none';\n // //Draw the foreground\n this.outputCtx!.drawImage(this.input!, 0, 0);\n\n if (\n this.enableSharpening &&\n this.output!.width > minVideoWidthForSharpening && // minimum and maximum resolution to enable sharpening filter\n this.output!.height > minVideoHeightForSharpening &&\n this.output!.width < maxVideoWidthForSharpening &&\n this.output!.height < maxVideoHeightForSharpening\n ) {\n this.sharpenFilter();\n }\n\n // //Draw the background\n this.drawSegmentedBackground();\n }\n\n private sharpenFilter() {\n // adding sharpening filter to each frame to improve edges and brightness\n // The basic idea is that you take the weighed sum of a rectangle of pixels from the source image and use that as the output value using convolution filter\n // It is applied intermediate output with black background and only mask data in frame\n // Filter currently used is 3 x 3 sharpening filter with values as shown:\n // [ 0, -1, 0,\n // -1, 5, -1,\n // 0, -1, 0 ]\n const outputImageData = this.outputCtx!.getImageData(0, 0, this.output!.width, this.output!.height);\n\n // filters you may try\n // [-1, -1, -1, -1, 9, -1, -1, -1, -1]\n //[0, -1, 0, -1, 5, -1, 0, -1, 0]\n const output = this.filters.convolute(outputImageData);\n\n this.outputCtx!.putImageData(output, 0, 0);\n }\n\n private drawPersonMask() {\n this.outputCtx!.drawImage(\n this.personMaskCanvas,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private drawSegmentedBackground() {\n this.outputCtx!.globalCompositeOperation = 'destination-over';\n this.outputCtx!.imageSmoothingEnabled = true;\n this.outputCtx!.imageSmoothingQuality = 'high';\n if (this.isVirtualBackground) {\n if (this.backgroundType === 'video' && this.backgroundVideo != null && this.backgroundVideo!.readyState >= 4) {\n this.fitVideoToBackground();\n } else if (this.backgroundType === 'image') {\n this.fitImageToBackground();\n } else if (this.backgroundType === 'gif') {\n if (this.giflocalCount > this.gifFrames[this.gifFramesIndex].delay / DEFAULT_DELAY) {\n this.gifFramesIndex++;\n if (this.gifFramesIndex >= this.gifFrames.length) {\n this.gifFramesIndex = 0;\n }\n this.giflocalCount = 0;\n } else {\n this.giflocalCount++;\n }\n this.fitGifToBackground();\n }\n } else {\n this.addBlurToBackground();\n }\n }\n\n private async runSegmentation(skipProcessing?: boolean) {\n if (this.tfLite) {\n // const start = performance.now();\n\n this.resizeInputData();\n await this.infer(skipProcessing);\n this.postProcessing();\n // const end = performance.now();\n // this.log(TAG,\"time taken\",end -start);\n }\n }\n\n private fitVideoToBackground() {\n this.fitData(this.backgroundVideo, this.backgroundVideo!.videoWidth, this.backgroundVideo!.videoHeight);\n }\n\n private fitImageToBackground() {\n this.fitData(this.backgroundImage, this.backgroundImage!.width, this.backgroundImage!.height);\n }\n\n private fitGifToBackground() {\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas!.width = dims.width;\n this.tempGifCanvas!.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n\n this.fitData(this.tempGifCanvas, this.gifFrameImageData!.width, this.gifFrameImageData!.height);\n }\n\n private fitData(data: any, dataWidth: number, dataHeight: number) {\n let inputWidth: any, inputHeight: any, xoffset: any, yoffset: any;\n if (dataWidth / dataHeight < this.imageAspectRatio) {\n inputWidth = dataWidth;\n inputHeight = dataWidth / this.imageAspectRatio;\n xoffset = 0;\n yoffset = (dataHeight - inputHeight) / 2;\n } else {\n inputHeight = dataHeight;\n inputWidth = dataHeight * this.imageAspectRatio;\n yoffset = 0;\n xoffset = (dataWidth - inputWidth) / 2;\n }\n this.outputCtx!.drawImage(\n data,\n xoffset,\n yoffset,\n inputWidth,\n inputHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private async addBlurToBackground() {\n let blurValue = '15px';\n if (this.input!.width <= 160) {\n blurValue = '5px';\n } else if (this.input!.width <= 320) {\n blurValue = '10px';\n } else if (this.input!.width <= 640) {\n blurValue = '15px';\n } else if (this.input!.width <= 960) {\n blurValue = '20px';\n } else if (this.input!.width <= 1280) {\n blurValue = '25px';\n } else if (this.input!.width <= 1920) {\n blurValue = '30px';\n }\n\n this.outputCtx!.filter = `blur(${blurValue})`;\n this.outputCtx!.drawImage(this.input!, 0, 0, this.output!.width, this.output!.height);\n }\n\n private initSharpenFilter(): any {\n this.filters.tmpCanvas = document.createElement('canvas');\n this.filters.tmpCtx = this.filters.tmpCanvas.getContext('2d');\n\n this.filters.createImageData = (w: number, h: number) => {\n return this.filters.tmpCtx.createImageData(w, h);\n };\n\n this.filters.convolute = (pixels: ImageData, weights = [0, -1, 0, -1, 5, -1, 0, -1, 0], opaque: boolean) => {\n const side = Math.round(Math.sqrt(weights.length));\n\n const halfSide = Math.floor(side / 2);\n const src = pixels.data;\n const sw = pixels.width;\n const sh = pixels.height;\n // pad output by the convolution matrix\n const w = sw;\n const h = sh;\n const output = this.filters.createImageData(w, h);\n const dst = output.data;\n // go through the destination image pixels\n const alphaFac = opaque ? 1 : 0;\n for (let y = 0; y < h; y = y + 1) {\n for (let x = 0; x < w; x = x + 1) {\n const dstOff = (y * w + x) * 4;\n\n if (src[dstOff + 3] === 0) {\n continue;\n } else if (x < w && y < h) {\n const sy = y;\n const sx = x;\n\n // calculate the weighed sum of the source image pixels that\n // fall under the convolution matrix\n let r = 0,\n g = 0,\n b = 0,\n a = 0;\n for (let cy = 0; cy < side; cy++) {\n for (let cx = 0; cx < side; cx++) {\n const scy = sy + cy - halfSide;\n const scx = sx + cx - halfSide;\n if (scy >= 0 && scy < sh && scx >= 0 && scx < sw) {\n const srcOff = (scy * sw + scx) * 4;\n const wt = weights[cy * side + cx];\n r += src[srcOff] * wt;\n g += src[srcOff + 1] * wt;\n b += src[srcOff + 2] * wt;\n a += src[srcOff + 3] * wt;\n }\n }\n }\n dst[dstOff] = r;\n dst[dstOff + 1] = g;\n dst[dstOff + 2] = b;\n dst[dstOff + 3] = a + alphaFac * (255 - a);\n }\n }\n }\n return output;\n };\n }\n}\n", "const pkg = require('../package.json');\nconst BASE_URL = `https://unpkg.com/${pkg.name}/src`;\nconst TAG = 'VBProcessor';\nconst TFLITE_JS_FILE = 'tflite/tflite.js';\nconst TFLITE_SIMD_JS_FILE = 'tflite/tflite-simd.js';\nconst MODEL_FILE_NAME = 'models/selfie_segmentation_landscape.tflite';\n\nconst loadScript = (src: string) => {\n return new Promise(function (resolve, reject) {\n const s = document.createElement('script');\n s.src = src;\n s.onload = resolve;\n s.onerror = reject;\n document.head.appendChild(s);\n });\n};\n\nconst loadTFLiteModel = async () => {\n let tfLite: any;\n let path = `${BASE_URL}/${TFLITE_SIMD_JS_FILE}`;\n await loadScript(path);\n try {\n //@ts-ignore\n tfLite = await createTFLiteSIMDModule();\n } catch {\n console.warn('SIMD not supported. You may experience poor virtual background effect.');\n path = `${BASE_URL}/${TFLITE_JS_FILE}`;\n await loadScript(path);\n // @ts-ignore\n tfLite = await createTFLiteModule();\n }\n return tfLite;\n};\n\nconst loadTFLite = async () => {\n const modelPath = `${BASE_URL}/${MODEL_FILE_NAME}`;\n const [tfLite, modelResponse] = await Promise.all([loadTFLiteModel(), fetch(modelPath)]);\n\n const model = await modelResponse.arrayBuffer();\n const modelBufferOffset = tfLite._getModelBufferMemoryOffset();\n tfLite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);\n tfLite._loadModel(model.byteLength);\n\n console.debug(TAG, 'Input memory offset:', tfLite._getInputMemoryOffset());\n console.debug(TAG, 'Input height:', tfLite._getInputHeight());\n console.debug(TAG, 'Input width:', tfLite._getInputWidth());\n console.debug(TAG, 'Input channels:', tfLite._getInputChannelCount());\n\n return tfLite;\n};\n\nexport { loadTFLite };\n", "/* eslint-disable complexity */\nimport { Results as MediaPipeResults, SelfieSegmentation } from '@mediapipe/selfie_segmentation';\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport { HMSBackgroundInput, HMSVirtualBackground, HMSVirtualBackgroundTypes } from './interfaces';\n\nexport class HMSVBPlugin implements HMSVideoPlugin {\n private TAG = '[HMSVBPlugin]';\n private background: HMSVirtualBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType: HMSVirtualBackgroundTypes = HMSVirtualBackgroundTypes.NONE;\n private segmentation!: SelfieSegmentation;\n private outputCanvas?: HTMLCanvasElement;\n private outputCtx?: CanvasRenderingContext2D | null;\n\n private gifFrames: any;\n private gifFramesIndex: number;\n private gifFrameImageData: any;\n private tempGifCanvas: HTMLCanvasElement;\n private tempGifContext: CanvasRenderingContext2D | null;\n private prevResults?: MediaPipeResults;\n private input?: HTMLCanvasElement;\n\n constructor(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n this.background = background;\n this.backgroundType = backgroundType;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n\n this.setBackground(this.background, this.backgroundType);\n this.log('Virtual background plugin initialised');\n }\n\n isSupported(): boolean {\n return this.checkSupport().isSupported;\n }\n\n isBlurSupported(): boolean {\n return 'filter' in CanvasRenderingContext2D.prototype;\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge', 'Safari'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return 'HMSVB';\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async init(): Promise<void> {\n if (!this.segmentation) {\n this.segmentation = new SelfieSegmentation({\n locateFile: (file: string) => {\n return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${file}`;\n },\n });\n this.segmentation.setOptions({ selfieMode: false, modelSelection: 1 });\n this.segmentation.onResults(this.handleResults);\n }\n }\n\n /**\n * For bgType HMSVirtualBackgroundTypes.IMAGE pass bg as an image element\n * For bgType HMSVirtualBackgroundTypes.VIDEO pass video\n * For bgType HMSVirtualBackgroundTypes.GIF pass the gif url\n * @param {HMSVirtualBackground} background\n * @param {HMSVirtualBackgroundTypes} backgroundType\n */\n async setBackground(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n if (!background) {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n this.prevResults = undefined;\n switch (backgroundType) {\n case HMSVirtualBackgroundTypes.NONE:\n case HMSVirtualBackgroundTypes.BLUR:\n this.background = background;\n this.backgroundType = backgroundType;\n break;\n case HMSVirtualBackgroundTypes.IMAGE:\n this.log('setting background to image', background);\n // eslint-disable-next-line no-case-declarations\n const img = await this.setImage(background as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.background = img;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n }\n break;\n case HMSVirtualBackgroundTypes.VIDEO:\n this.log('setting background to video', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as HTMLVideoElement;\n this.background.crossOrigin = 'anonymous';\n this.background.muted = true;\n this.background.loop = true;\n this.background.playsInline = true;\n this.background.oncanplaythrough = async () => {\n if (this.background && this.background instanceof HTMLVideoElement) {\n try {\n await this.background.play();\n this.backgroundType = HMSVirtualBackgroundTypes.VIDEO;\n } catch (e) {\n this.log('failed to play background', background);\n }\n }\n };\n break;\n case HMSVirtualBackgroundTypes.CANVAS:\n this.background = background;\n this.backgroundType = HMSVirtualBackgroundTypes.CANVAS;\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.log('setting gif to background', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as string;\n this.gifFrames = await this.loadGIF(this.background);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = HMSVirtualBackgroundTypes.GIF;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n break;\n default:\n this.log(\n `backgroundType did not match with any of the supported background types - ${HMSVirtualBackgroundTypes}`,\n );\n }\n }\n\n getBackground() {\n return this.background;\n }\n\n stop(): void {\n if (this.backgroundType !== HMSVirtualBackgroundTypes.BLUR && this.background !== HMSVirtualBackgroundTypes.NONE) {\n this.segmentation?.reset();\n }\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.background = HMSVirtualBackgroundTypes.NONE;\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n }\n\n async processVideoFrame(input: HTMLCanvasElement, output: HTMLCanvasElement, skipProcessing?: boolean) {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n this.input = input;\n output.width = input.width;\n output.height = input.height;\n this.outputCanvas = output;\n this.outputCtx = output.getContext('2d');\n if (skipProcessing && this.prevResults) {\n this.handleResults(this.prevResults);\n return;\n }\n if (this.backgroundType === HMSVirtualBackgroundTypes.NONE) {\n this.outputCtx?.drawImage(input, 0, 0, input.width, input.height);\n return;\n }\n await this.segmentation.send({ image: input });\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private handleResults = (results: MediaPipeResults) => {\n if (!this.outputCanvas || !this.outputCtx) {\n return;\n }\n this.outputCtx.save();\n this.outputCtx.clearRect(0, 0, this.outputCanvas.width, this.outputCanvas.height);\n switch (this.backgroundType) {\n case HMSVirtualBackgroundTypes.IMAGE:\n case HMSVirtualBackgroundTypes.CANVAS:\n case HMSVirtualBackgroundTypes.VIDEO:\n this.renderBackground(results, this.background as HMSBackgroundInput);\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.renderGIF(results);\n break;\n case HMSVirtualBackgroundTypes.BLUR:\n this.renderBlur(results);\n break;\n }\n this.outputCtx.restore();\n this.prevResults = results;\n };\n\n private loadGIF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(...data: any[]) {\n console.debug(this.TAG, ...data);\n }\n\n private renderBackground = (results: MediaPipeResults, background: HMSBackgroundInput) => {\n if (\n !this.input ||\n !this.outputCanvas ||\n !this.outputCtx ||\n this.backgroundType === HMSVirtualBackgroundTypes.NONE ||\n this.backgroundType === HMSVirtualBackgroundTypes.BLUR\n ) {\n return;\n }\n this.outputCtx.filter = 'none';\n this.outputCtx.imageSmoothingEnabled = true;\n this.outputCtx.imageSmoothingQuality = 'high';\n // Only overwrite existing pixels.\n this.outputCtx.globalCompositeOperation = 'source-out';\n const bgWidth = background instanceof HTMLVideoElement ? background.videoWidth : background.width;\n const bgHeight = background instanceof HTMLVideoElement ? background.videoHeight : background.height;\n\n this.outputCtx.drawImage(\n background,\n 0,\n 0,\n bgWidth,\n bgHeight,\n 0,\n 0,\n this.outputCanvas.width,\n this.outputCanvas.height,\n );\n this.outputCtx.globalCompositeOperation = 'destination-out';\n this.outputCtx.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n // Only overwrite missing pixels.\n this.outputCtx.globalCompositeOperation = 'destination-atop';\n this.outputCtx.drawImage(this.input, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n };\n\n private renderBlur(results: MediaPipeResults) {\n if (!this.outputCanvas || !this.outputCtx || this.backgroundType !== HMSVirtualBackgroundTypes.BLUR) {\n return;\n }\n this.outputCtx!.filter = 'none';\n this.outputCtx!.globalCompositeOperation = 'source-out';\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.globalCompositeOperation = 'destination-atop';\n this.outputCtx?.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.filter = `blur(${Math.floor(this.outputCanvas.width / 160) * 5}px)`;\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n }\n\n private renderGIF(results: MediaPipeResults) {\n if (\n !this.outputCanvas ||\n !this.outputCtx ||\n !this.tempGifContext ||\n this.backgroundType !== HMSVirtualBackgroundTypes.GIF\n ) {\n return;\n }\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas.width = dims.width;\n this.tempGifCanvas.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n this.gifFramesIndex = (this.gifFramesIndex + 1) % this.gifFrames.length;\n this.renderBackground(results, this.tempGifCanvas);\n }\n}\n\nexport { HMSVirtualBackgroundTypes };\n", "export type HMSBackgroundInput = HTMLImageElement | HTMLVideoElement | HTMLCanvasElement;\nexport enum HMSVirtualBackgroundTypes {\n BLUR = 'blur',\n NONE = 'none',\n GIF = 'gif',\n IMAGE = 'image',\n VIDEO = 'video',\n CANVAS = 'canvas',\n}\n\nexport type HMSVirtualBackground =\n | string\n | HMSVirtualBackgroundTypes.BLUR\n | HMSVirtualBackgroundTypes.NONE\n | HMSBackgroundInput;\n", "import { tsvb } from 'effects-sdk';\nimport { HMSMediaStreamPlugin } from '@100mslive/hms-video-store';\nimport { EFFECTS_SDK_ASSETS } from './constants';\nimport { HMSVirtualBackgroundTypes } from './interfaces';\n\nexport type HMSEffectsBackground = string | MediaStream | MediaStreamTrack | HTMLVideoElement;\n\nexport class HMSEffectsPlugin implements HMSMediaStreamPlugin {\n private effects: tsvb;\n // Ranges from 0 to 1, inclusive\n private blurAmount = 0;\n private background: HMSEffectsBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType = HMSVirtualBackgroundTypes.NONE;\n private preset: 'balanced' | 'quality' = 'balanced';\n private initialised = false;\n private intervalId: NodeJS.Timer | null = null;\n private onInit;\n private onResolutionChangeCallback?: (width: number, height: number) => void;\n private canvas: HTMLCanvasElement;\n\n constructor(effectsSDKKey: string, onInit?: () => void) {\n this.effects = new tsvb(effectsSDKKey);\n this.onInit = onInit;\n this.effects.config({\n sdk_url: EFFECTS_SDK_ASSETS,\n models: {\n colorcorrector: '',\n facedetector: '',\n lowlighter: '',\n },\n wasmPaths: {\n 'ort-wasm.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm.wasm`,\n 'ort-wasm-simd.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd.wasm`,\n },\n });\n this.canvas = document.createElement('canvas');\n this.effects.onError(err => {\n // currently logging info type messages as well\n if (!err.type || err.type === 'error') {\n console.error('[HMSEffectsPlugin]', err);\n }\n });\n this.effects.cache();\n this.effects.onReady = () => {\n if (this.effects) {\n this.initialised = true;\n this.onInit?.();\n this.effects.run();\n this.effects.setBackgroundFitMode('fill');\n this.effects.setSegmentationPreset(this.preset);\n this.applyEffect();\n }\n };\n }\n\n getName(): string {\n return 'HMSEffects';\n }\n\n private executeAfterInit(callback: () => void) {\n if (this.initialised) {\n callback();\n }\n\n if (this.intervalId !== null) {\n clearInterval(this.intervalId);\n }\n this.intervalId = setInterval(() => {\n if (this.initialised) {\n clearInterval(this.intervalId!);\n callback();\n }\n }, 100);\n }\n\n removeBlur() {\n this.blurAmount = 0;\n this.executeAfterInit(() => {\n this.effects.clearBlur();\n });\n }\n\n removeBackground() {\n this.background = '';\n this.executeAfterInit(() => {\n this.effects.clearBackground();\n });\n }\n\n /**\n * @param blur ranges between 0 and 1\n */\n setBlur(blur: number) {\n this.blurAmount = blur;\n this.backgroundType = HMSVirtualBackgroundTypes.BLUR;\n this.removeBackground();\n this.executeAfterInit(() => {\n this.effects.setBlur(this.blurAmount);\n });\n }\n\n /**\n * @param preset can be 'quality' or 'balanced'. The 'quality' preset has better quality but higher CPU usage than 'balanced'\n */\n async setPreset(preset: 'quality' | 'balanced') {\n this.preset = preset;\n return new Promise((resolve, reject) => {\n this.executeAfterInit(() => {\n this.effects.setSegmentationPreset(this.preset).then(resolve).catch(reject);\n });\n });\n }\n\n onResolutionChange(callback: (width: number, height: number) => void) {\n this.onResolutionChangeCallback = callback;\n }\n\n getPreset() {\n return this.preset;\n }\n\n removeEffects() {\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.removeBackground();\n this.removeBlur();\n }\n\n setBackground(url: HMSEffectsBackground) {\n this.background = url;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n this.removeBlur();\n this.executeAfterInit(() => {\n this.effects.setBackground(this.background);\n });\n }\n\n getBlurAmount() {\n return this.blurAmount;\n }\n\n getBackground() {\n return this.background || this.backgroundType;\n }\n\n private updateCanvas(stream: MediaStream) {\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.canvas.width = width!;\n this.canvas.height = height!;\n this.effects.useStream(stream);\n this.effects.toCanvas(this.canvas);\n }\n\n apply(stream: MediaStream): MediaStream {\n this.effects.clear();\n this.applyEffect();\n this.effects.onChangeInputResolution(() => {\n this.updateCanvas(stream);\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.onResolutionChangeCallback?.(width!, height!);\n });\n this.updateCanvas(stream);\n return this.canvas.captureStream(30) || stream;\n }\n\n stop() {\n this.removeEffects();\n this.executeAfterInit(() => {\n this.effects.stop();\n });\n }\n\n private applyEffect() {\n if (this.blurAmount) {\n this.setBlur(this.blurAmount);\n } else if (this.background) {\n this.setBackground(this.background);\n }\n }\n}\n", "export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/3.4.3/';\n"],
4
+ "sourcesContent": ["{\n \"version\": \"1.13.28\",\n \"license\": \"MIT\",\n \"name\": \"@100mslive/hms-virtual-background\",\n \"author\": \"100ms\",\n \"module\": \"dist/esm/index.js\",\n \"main\": \"dist/cjs/index.js\",\n \"typings\": \"dist/index.d.ts\",\n \"typesVersions\": {\n \"*\": {\n \"hmsvbplugin\": [\n \"./dist/HMSVBPlugin.d.ts\"\n ],\n \"hmseffectsplugin\": [\n \"./dist/HMSEffectsPlugin.d.ts\"\n ]\n }\n },\n \"exports\": {\n \".\": {\n \"import\": \"./dist/esm/index.js\",\n \"require\": \"./dist/cjs/index.js\",\n \"default\": \"./dist/esm/index.js\",\n \"types\": \"./dist/index.d.ts\"\n },\n \"./hmsvbplugin\": {\n \"import\": \"./dist/esm/HMSVBPlugin.js\",\n \"require\": \"./dist/cjs/HMSVBPlugin.js\",\n \"default\": \"./dist/esm/HMSVBPlugin.js\",\n \"types\": \"./dist/HMSVBPlugin.d.ts\"\n },\n \"./hmseffectsplugin\": {\n \"import\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"default\": \"./dist/esm/HMSEffectsPlugin.js\",\n \"require\": \"./dist/cjs/HMSEffectsPlugin.js\",\n \"types\": \"./dist/HMSEffectsPlugin.d.ts\"\n }\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"https://github.com/100mslive/web-sdks.git\",\n \"directory\": \"packages/hms-virtual-background\"\n },\n \"files\": [\n \"dist\",\n \"src/tflite\",\n \"src/models\"\n ],\n \"scripts\": {\n \"start\": \"concurrently \\\"yarn dev\\\" \\\"yarn types\\\"\",\n \"dev\": \"node ../../scripts/dev\",\n \"build:only\": \"node ../../scripts/build\",\n \"build\": \"yarn build:only && yarn types:build\",\n \"types\": \"tsc -w\",\n \"types:build\": \"tsc -p tsconfig.json\",\n \"test\": \"jest --maxWorkers=1 --passWithNoTests\",\n \"lint\": \"eslint -c ../../.eslintrc .\",\n \"lint:fix\": \"yarn lint --fix\",\n \"prepare\": \"yarn build\",\n \"size\": \"size-limit\",\n \"analyze\": \"size-limit --why\",\n \"format\": \"prettier --write src/**/*.ts\"\n },\n \"peerDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.28\"\n },\n \"devDependencies\": {\n \"@100mslive/hms-video-store\": \"0.12.28\"\n },\n \"dependencies\": {\n \"@mediapipe/selfie_segmentation\": \"^0.1.1632777926\",\n \"@tensorflow-models/body-segmentation\": \"^1.0.1\",\n \"@tensorflow/tfjs-backend-webgl\": \"^3.3.0\",\n \"@tensorflow/tfjs-converter\": \"^3.19.0\",\n \"@tensorflow/tfjs-core\": \"^3.19.0\",\n \"@webassemblyjs/helper-wasm-bytecode\": \"1.11.1\",\n \"@webassemblyjs/wasm-gen\": \"1.11.1\",\n \"effects-sdk\": \"3.4.3\",\n \"gifuct-js\": \"^2.1.2\",\n \"wasm-check\": \"^2.0.2\"\n },\n \"eslintIgnore\": [\n \"tflite.js\",\n \"tflite-simd.js\",\n \"tflite.wasm\",\n \"tflite-simd.wasm\",\n \"defineTFLite.ts\",\n \"importing.test.ts\"\n ],\n \"gitHead\": \"c4d7367340117e66de386fc7aaa9b5e95abd19ed\"\n}\n", "/* eslint-disable complexity */\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport '@tensorflow/tfjs-backend-webgl';\nimport { loadTFLite } from './defineTFLite';\n\nconst TAG = 'VBProcessor';\nconst DEFAULT_DELAY = 33;\nconst pkg = require('../package.json');\n\nconst minVideoWidthForSharpening = 214;\nconst maxVideoWidthForSharpening = 855;\nconst minVideoHeightForSharpening = 120;\nconst maxVideoHeightForSharpening = 720;\n\nexport class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {\n background: string | HTMLImageElement;\n personMaskWidth: number;\n personMaskHeight: number;\n isVirtualBackground: boolean;\n backgroundImage: HTMLImageElement | null;\n backgroundVideo: HTMLVideoElement | null;\n backgroundType = 'none';\n loadModelCalled: boolean;\n blurValue: any;\n tfLite: any;\n tfLitePromise: any;\n modelName: string;\n\n input: HTMLCanvasElement | null;\n output: HTMLCanvasElement | null;\n outputCtx: CanvasRenderingContext2D | null;\n timerID: number;\n imageAspectRatio: number;\n\n personMaskPixelCount: number;\n personMask: ImageData;\n personMaskCanvas: HTMLCanvasElement;\n personMaskCtx: any;\n filters: any;\n enableSharpening?: boolean | false;\n\n gifFrames: any;\n gifFramesIndex: number;\n gifFrameImageData: any;\n tempGifCanvas: HTMLCanvasElement;\n tempGifContext: any;\n giflocalCount: number;\n\n constructor(background: string, enableSharpening = false) {\n this.background = background;\n this.enableSharpening = enableSharpening;\n\n this.backgroundImage = null;\n this.backgroundVideo = null;\n\n this.personMaskWidth = 256;\n this.personMaskHeight = 144;\n this.isVirtualBackground = false;\n this.blurValue = '10px';\n this.loadModelCalled = false;\n this.tfLite = null;\n this.modelName = 'landscape-segmentation';\n\n this.outputCtx = null;\n this.input = null;\n this.output = null;\n this.timerID = 0;\n this.imageAspectRatio = 1;\n\n this.personMaskPixelCount = this.personMaskWidth * this.personMaskHeight;\n this.personMask = new ImageData(this.personMaskWidth, this.personMaskHeight);\n this.personMaskCanvas = document.createElement('canvas');\n this.personMaskCanvas.width = this.personMaskWidth;\n this.personMaskCanvas.height = this.personMaskHeight;\n this.personMaskCtx = this.personMaskCanvas.getContext('2d');\n\n this.filters = {};\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n this.giflocalCount = 0;\n this.enableSharpening = enableSharpening;\n\n this.log(TAG, 'Virtual Background plugin created');\n this.setBackground(this.background);\n }\n\n async init(): Promise<void> {\n if (!this.loadModelCalled) {\n this.log(TAG, 'PREVIOUS LOADED MODEL IS ', this.tfLite);\n this.loadModelCalled = true;\n this.tfLitePromise = loadTFLite();\n this.tfLite = await this.tfLitePromise;\n } else {\n //either it is loading or loaded\n await this.tfLitePromise;\n }\n if (this.enableSharpening) {\n this.initSharpenFilter();\n }\n }\n\n /*\n @depreceated\n */\n isSupported(): boolean {\n //support chrome, firefox, edge TODO: check this\n return (\n navigator.userAgent.indexOf('Chrome') !== -1 ||\n navigator.userAgent.indexOf('Firefox') !== -1 ||\n navigator.userAgent.indexOf('Edg') !== -1 ||\n navigator.userAgent.indexOf('Edge') !== -1\n );\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return pkg.name;\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async setBackground(path?: string | HTMLImageElement | HTMLVideoElement) {\n if (path !== '') {\n if (path === 'none') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'none';\n this.backgroundType = 'none';\n this.isVirtualBackground = false;\n } else if (path === 'blur') {\n this.log(TAG, 'setting background to :', path);\n this.background = 'blur';\n this.backgroundType = 'blur';\n this.isVirtualBackground = false;\n } else if (path instanceof HTMLImageElement) {\n this.log('setting background to image', path);\n const img = await this.setImage(path as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.isVirtualBackground = true;\n this.backgroundImage = img;\n this.backgroundType = 'image';\n }\n } else if (path instanceof HTMLVideoElement) {\n this.log('setting background to video', path);\n this.backgroundVideo = path as HTMLVideoElement;\n this.backgroundVideo.crossOrigin = 'anonymous';\n this.backgroundVideo.muted = true;\n this.backgroundVideo.loop = true;\n this.backgroundVideo.oncanplaythrough = async () => {\n if (this.backgroundVideo != null) {\n await this.backgroundVideo!.play();\n this.isVirtualBackground = true;\n this.backgroundType = 'video';\n }\n };\n } else {\n console.log('setting gif to background');\n this.gifFrames = await this.setGiF(path as string);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = 'gif';\n this.isVirtualBackground = true;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n }\n\n stop(): void {\n if (this.isVirtualBackground) {\n this.backgroundImage?.removeAttribute('src');\n this.backgroundVideo?.removeAttribute('src');\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.loop = false;\n this.backgroundVideo = null;\n }\n }\n if (this.outputCtx) {\n this.outputCtx!.fillStyle = `rgb(0, 0, 0)`;\n this.outputCtx!.fillRect(0, 0, this.output!.width, this.output!.height);\n }\n\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.giflocalCount = 0;\n this.gifFramesIndex = 0;\n }\n\n processVideoFrame(\n input: HTMLCanvasElement,\n output: HTMLCanvasElement,\n skipProcessing?: boolean,\n ): Promise<void> | void {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n\n this.input = input;\n this.output = output;\n\n const ctx = output.getContext('2d');\n if (ctx!.canvas.width !== input.width) {\n ctx!.canvas.width = input.width;\n }\n if (ctx!.canvas.height !== input.height) {\n ctx!.canvas.height = input.height;\n }\n\n if (this.backgroundType === 'video') {\n this.backgroundVideo!.width = input.width;\n this.backgroundVideo!.height = input.height;\n }\n\n this.outputCtx = ctx!;\n this.imageAspectRatio = input.width / input.height;\n if (this.imageAspectRatio <= 0) {\n throw new Error('Invalid input width/height');\n }\n\n const process = async () => {\n await this.runSegmentation(skipProcessing);\n };\n\n if (this.background === 'none' && !this.isVirtualBackground) {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n this.outputCtx!.drawImage(input, 0, 0, input.width, input.height);\n } else {\n process();\n }\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private setGiF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(tag: string, ...data: any[]) {\n console.info(tag, ...data);\n }\n\n private resizeInputData() {\n this.personMaskCtx!.drawImage(\n this.input,\n 0,\n 0,\n this.input!.width,\n this.input!.height,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n );\n\n const imageData = this.personMaskCtx!.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight);\n //\n const inputMemoryOffset = this.tfLite._getInputMemoryOffset() / 4;\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3] = imageData.data[i * 4] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 1] = imageData.data[i * 4 + 1] / 255;\n this.tfLite.HEAPF32[inputMemoryOffset + i * 3 + 2] = imageData.data[i * 4 + 2] / 255;\n }\n }\n private infer(skipProcessing?: boolean) {\n if (!skipProcessing) {\n this.tfLite._runInference();\n }\n const outputMemoryOffset = this.tfLite._getOutputMemoryOffset() / 4;\n\n for (let i = 0; i < this.personMaskPixelCount; i++) {\n if (this.modelName === 'meet') {\n const background = this.tfLite.HEAPF32[outputMemoryOffset + i * 2];\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i * 2 + 1];\n const shift = Math.max(background, person);\n const backgroundExp = Math.exp(background - shift);\n const personExp = Math.exp(person - shift);\n // Sets only the alpha component of each pixel.\n this.personMask.data[i * 4 + 3] = (255 * personExp) / (backgroundExp + personExp);\n } else if (this.modelName === 'landscape-segmentation') {\n const person = this.tfLite.HEAPF32[outputMemoryOffset + i];\n this.personMask.data[i * 4 + 3] = 255 * person;\n }\n }\n\n this.personMaskCtx!.putImageData(this.personMask, 0, 0);\n }\n\n private postProcessing() {\n this.outputCtx!.globalCompositeOperation = 'copy';\n this.outputCtx!.filter = 'none';\n\n if (this.isVirtualBackground) {\n this.outputCtx!.filter = 'blur(4px)';\n } else {\n this.outputCtx!.filter = 'blur(8px)';\n }\n this.drawPersonMask();\n this.outputCtx!.globalCompositeOperation = 'source-in';\n this.outputCtx!.filter = 'none';\n // //Draw the foreground\n this.outputCtx!.drawImage(this.input!, 0, 0);\n\n if (\n this.enableSharpening &&\n this.output!.width > minVideoWidthForSharpening && // minimum and maximum resolution to enable sharpening filter\n this.output!.height > minVideoHeightForSharpening &&\n this.output!.width < maxVideoWidthForSharpening &&\n this.output!.height < maxVideoHeightForSharpening\n ) {\n this.sharpenFilter();\n }\n\n // //Draw the background\n this.drawSegmentedBackground();\n }\n\n private sharpenFilter() {\n // adding sharpening filter to each frame to improve edges and brightness\n // The basic idea is that you take the weighed sum of a rectangle of pixels from the source image and use that as the output value using convolution filter\n // It is applied intermediate output with black background and only mask data in frame\n // Filter currently used is 3 x 3 sharpening filter with values as shown:\n // [ 0, -1, 0,\n // -1, 5, -1,\n // 0, -1, 0 ]\n const outputImageData = this.outputCtx!.getImageData(0, 0, this.output!.width, this.output!.height);\n\n // filters you may try\n // [-1, -1, -1, -1, 9, -1, -1, -1, -1]\n //[0, -1, 0, -1, 5, -1, 0, -1, 0]\n const output = this.filters.convolute(outputImageData);\n\n this.outputCtx!.putImageData(output, 0, 0);\n }\n\n private drawPersonMask() {\n this.outputCtx!.drawImage(\n this.personMaskCanvas,\n 0,\n 0,\n this.personMaskWidth,\n this.personMaskHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private drawSegmentedBackground() {\n this.outputCtx!.globalCompositeOperation = 'destination-over';\n this.outputCtx!.imageSmoothingEnabled = true;\n this.outputCtx!.imageSmoothingQuality = 'high';\n if (this.isVirtualBackground) {\n if (this.backgroundType === 'video' && this.backgroundVideo != null && this.backgroundVideo!.readyState >= 4) {\n this.fitVideoToBackground();\n } else if (this.backgroundType === 'image') {\n this.fitImageToBackground();\n } else if (this.backgroundType === 'gif') {\n if (this.giflocalCount > this.gifFrames[this.gifFramesIndex].delay / DEFAULT_DELAY) {\n this.gifFramesIndex++;\n if (this.gifFramesIndex >= this.gifFrames.length) {\n this.gifFramesIndex = 0;\n }\n this.giflocalCount = 0;\n } else {\n this.giflocalCount++;\n }\n this.fitGifToBackground();\n }\n } else {\n this.addBlurToBackground();\n }\n }\n\n private async runSegmentation(skipProcessing?: boolean) {\n if (this.tfLite) {\n // const start = performance.now();\n\n this.resizeInputData();\n await this.infer(skipProcessing);\n this.postProcessing();\n // const end = performance.now();\n // this.log(TAG,\"time taken\",end -start);\n }\n }\n\n private fitVideoToBackground() {\n this.fitData(this.backgroundVideo, this.backgroundVideo!.videoWidth, this.backgroundVideo!.videoHeight);\n }\n\n private fitImageToBackground() {\n this.fitData(this.backgroundImage, this.backgroundImage!.width, this.backgroundImage!.height);\n }\n\n private fitGifToBackground() {\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas!.width = dims.width;\n this.tempGifCanvas!.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n\n this.fitData(this.tempGifCanvas, this.gifFrameImageData!.width, this.gifFrameImageData!.height);\n }\n\n private fitData(data: any, dataWidth: number, dataHeight: number) {\n let inputWidth: any, inputHeight: any, xoffset: any, yoffset: any;\n if (dataWidth / dataHeight < this.imageAspectRatio) {\n inputWidth = dataWidth;\n inputHeight = dataWidth / this.imageAspectRatio;\n xoffset = 0;\n yoffset = (dataHeight - inputHeight) / 2;\n } else {\n inputHeight = dataHeight;\n inputWidth = dataHeight * this.imageAspectRatio;\n yoffset = 0;\n xoffset = (dataWidth - inputWidth) / 2;\n }\n this.outputCtx!.drawImage(\n data,\n xoffset,\n yoffset,\n inputWidth,\n inputHeight,\n 0,\n 0,\n this.output!.width,\n this.output!.height,\n );\n }\n\n private async addBlurToBackground() {\n let blurValue = '15px';\n if (this.input!.width <= 160) {\n blurValue = '5px';\n } else if (this.input!.width <= 320) {\n blurValue = '10px';\n } else if (this.input!.width <= 640) {\n blurValue = '15px';\n } else if (this.input!.width <= 960) {\n blurValue = '20px';\n } else if (this.input!.width <= 1280) {\n blurValue = '25px';\n } else if (this.input!.width <= 1920) {\n blurValue = '30px';\n }\n\n this.outputCtx!.filter = `blur(${blurValue})`;\n this.outputCtx!.drawImage(this.input!, 0, 0, this.output!.width, this.output!.height);\n }\n\n private initSharpenFilter(): any {\n this.filters.tmpCanvas = document.createElement('canvas');\n this.filters.tmpCtx = this.filters.tmpCanvas.getContext('2d');\n\n this.filters.createImageData = (w: number, h: number) => {\n return this.filters.tmpCtx.createImageData(w, h);\n };\n\n this.filters.convolute = (pixels: ImageData, weights = [0, -1, 0, -1, 5, -1, 0, -1, 0], opaque: boolean) => {\n const side = Math.round(Math.sqrt(weights.length));\n\n const halfSide = Math.floor(side / 2);\n const src = pixels.data;\n const sw = pixels.width;\n const sh = pixels.height;\n // pad output by the convolution matrix\n const w = sw;\n const h = sh;\n const output = this.filters.createImageData(w, h);\n const dst = output.data;\n // go through the destination image pixels\n const alphaFac = opaque ? 1 : 0;\n for (let y = 0; y < h; y = y + 1) {\n for (let x = 0; x < w; x = x + 1) {\n const dstOff = (y * w + x) * 4;\n\n if (src[dstOff + 3] === 0) {\n continue;\n } else if (x < w && y < h) {\n const sy = y;\n const sx = x;\n\n // calculate the weighed sum of the source image pixels that\n // fall under the convolution matrix\n let r = 0,\n g = 0,\n b = 0,\n a = 0;\n for (let cy = 0; cy < side; cy++) {\n for (let cx = 0; cx < side; cx++) {\n const scy = sy + cy - halfSide;\n const scx = sx + cx - halfSide;\n if (scy >= 0 && scy < sh && scx >= 0 && scx < sw) {\n const srcOff = (scy * sw + scx) * 4;\n const wt = weights[cy * side + cx];\n r += src[srcOff] * wt;\n g += src[srcOff + 1] * wt;\n b += src[srcOff + 2] * wt;\n a += src[srcOff + 3] * wt;\n }\n }\n }\n dst[dstOff] = r;\n dst[dstOff + 1] = g;\n dst[dstOff + 2] = b;\n dst[dstOff + 3] = a + alphaFac * (255 - a);\n }\n }\n }\n return output;\n };\n }\n}\n", "const pkg = require('../package.json');\nconst BASE_URL = `https://unpkg.com/${pkg.name}/src`;\nconst TAG = 'VBProcessor';\nconst TFLITE_JS_FILE = 'tflite/tflite.js';\nconst TFLITE_SIMD_JS_FILE = 'tflite/tflite-simd.js';\nconst MODEL_FILE_NAME = 'models/selfie_segmentation_landscape.tflite';\n\nconst loadScript = (src: string) => {\n return new Promise(function (resolve, reject) {\n const s = document.createElement('script');\n s.src = src;\n s.onload = resolve;\n s.onerror = reject;\n document.head.appendChild(s);\n });\n};\n\nconst loadTFLiteModel = async () => {\n let tfLite: any;\n let path = `${BASE_URL}/${TFLITE_SIMD_JS_FILE}`;\n await loadScript(path);\n try {\n //@ts-ignore\n tfLite = await createTFLiteSIMDModule();\n } catch {\n console.warn('SIMD not supported. You may experience poor virtual background effect.');\n path = `${BASE_URL}/${TFLITE_JS_FILE}`;\n await loadScript(path);\n // @ts-ignore\n tfLite = await createTFLiteModule();\n }\n return tfLite;\n};\n\nconst loadTFLite = async () => {\n const modelPath = `${BASE_URL}/${MODEL_FILE_NAME}`;\n const [tfLite, modelResponse] = await Promise.all([loadTFLiteModel(), fetch(modelPath)]);\n\n const model = await modelResponse.arrayBuffer();\n const modelBufferOffset = tfLite._getModelBufferMemoryOffset();\n tfLite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);\n tfLite._loadModel(model.byteLength);\n\n console.debug(TAG, 'Input memory offset:', tfLite._getInputMemoryOffset());\n console.debug(TAG, 'Input height:', tfLite._getInputHeight());\n console.debug(TAG, 'Input width:', tfLite._getInputWidth());\n console.debug(TAG, 'Input channels:', tfLite._getInputChannelCount());\n\n return tfLite;\n};\n\nexport { loadTFLite };\n", "/* eslint-disable complexity */\nimport { Results as MediaPipeResults, SelfieSegmentation } from '@mediapipe/selfie_segmentation';\nimport { decompressFrames, parseGIF } from 'gifuct-js';\nimport {\n HMSPluginSupportResult,\n HMSPluginUnsupportedTypes,\n HMSVideoPlugin,\n HMSVideoPluginType,\n} from '@100mslive/hms-video-store';\nimport { HMSBackgroundInput, HMSVirtualBackground, HMSVirtualBackgroundTypes } from './interfaces';\n\nexport class HMSVBPlugin implements HMSVideoPlugin {\n private TAG = '[HMSVBPlugin]';\n private background: HMSVirtualBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType: HMSVirtualBackgroundTypes = HMSVirtualBackgroundTypes.NONE;\n private segmentation!: SelfieSegmentation;\n private outputCanvas?: HTMLCanvasElement;\n private outputCtx?: CanvasRenderingContext2D | null;\n\n private gifFrames: any;\n private gifFramesIndex: number;\n private gifFrameImageData: any;\n private tempGifCanvas: HTMLCanvasElement;\n private tempGifContext: CanvasRenderingContext2D | null;\n private prevResults?: MediaPipeResults;\n private input?: HTMLCanvasElement;\n\n constructor(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n this.background = background;\n this.backgroundType = backgroundType;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.gifFrameImageData = null;\n this.tempGifCanvas = document.createElement('canvas');\n this.tempGifContext = this.tempGifCanvas.getContext('2d');\n\n this.setBackground(this.background, this.backgroundType);\n this.log('Virtual background plugin initialised');\n }\n\n isSupported(): boolean {\n return this.checkSupport().isSupported;\n }\n\n isBlurSupported(): boolean {\n return 'filter' in CanvasRenderingContext2D.prototype;\n }\n\n checkSupport(): HMSPluginSupportResult {\n const browserResult = {} as HMSPluginSupportResult;\n if (['Chrome', 'Firefox', 'Edg', 'Edge', 'Safari'].some(value => navigator.userAgent.indexOf(value) !== -1)) {\n browserResult.isSupported = true;\n } else {\n browserResult.isSupported = false;\n browserResult.errType = HMSPluginUnsupportedTypes.PLATFORM_NOT_SUPPORTED;\n browserResult.errMsg = 'browser not supported for plugin, see docs';\n }\n\n return browserResult;\n }\n\n getName(): string {\n return 'HMSVB';\n }\n\n getPluginType(): HMSVideoPluginType {\n return HMSVideoPluginType.TRANSFORM;\n }\n\n async init(): Promise<void> {\n if (!this.segmentation) {\n this.segmentation = new SelfieSegmentation({\n locateFile: (file: string) => {\n return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1/${file}`;\n },\n });\n this.segmentation.setOptions({ selfieMode: false, modelSelection: 1 });\n this.segmentation.onResults(this.handleResults);\n }\n }\n\n /**\n * For bgType HMSVirtualBackgroundTypes.IMAGE pass bg as an image element\n * For bgType HMSVirtualBackgroundTypes.VIDEO pass video\n * For bgType HMSVirtualBackgroundTypes.GIF pass the gif url\n * @param {HMSVirtualBackground} background\n * @param {HMSVirtualBackgroundTypes} backgroundType\n */\n async setBackground(background: HMSVirtualBackground, backgroundType: HMSVirtualBackgroundTypes) {\n if (!background) {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n this.prevResults = undefined;\n switch (backgroundType) {\n case HMSVirtualBackgroundTypes.NONE:\n case HMSVirtualBackgroundTypes.BLUR:\n this.background = background;\n this.backgroundType = backgroundType;\n break;\n case HMSVirtualBackgroundTypes.IMAGE:\n this.log('setting background to image', background);\n // eslint-disable-next-line no-case-declarations\n const img = await this.setImage(background as HTMLImageElement);\n if (!img || !img.complete || !img.naturalHeight) {\n throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');\n } else {\n this.background = img;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n }\n break;\n case HMSVirtualBackgroundTypes.VIDEO:\n this.log('setting background to video', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as HTMLVideoElement;\n this.background.crossOrigin = 'anonymous';\n this.background.muted = true;\n this.background.loop = true;\n this.background.playsInline = true;\n this.background.oncanplaythrough = async () => {\n if (this.background && this.background instanceof HTMLVideoElement) {\n try {\n await this.background.play();\n this.backgroundType = HMSVirtualBackgroundTypes.VIDEO;\n } catch (e) {\n this.log('failed to play background', background);\n }\n }\n };\n break;\n case HMSVirtualBackgroundTypes.CANVAS:\n this.background = background;\n this.backgroundType = HMSVirtualBackgroundTypes.CANVAS;\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.log('setting gif to background', background);\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.background = background as string;\n this.gifFrames = await this.loadGIF(this.background);\n if (this.gifFrames != null && this.gifFrames.length > 0) {\n this.backgroundType = HMSVirtualBackgroundTypes.GIF;\n } else {\n throw new Error('Invalid background supplied, see the docs to check supported background type');\n }\n break;\n default:\n this.log(\n `backgroundType did not match with any of the supported background types - ${HMSVirtualBackgroundTypes}`,\n );\n }\n }\n\n getBackground() {\n return this.background;\n }\n\n stop(): void {\n if (this.backgroundType !== HMSVirtualBackgroundTypes.BLUR && this.background !== HMSVirtualBackgroundTypes.NONE) {\n this.segmentation?.reset();\n }\n //gif related\n this.gifFrameImageData = null;\n this.gifFrames = null;\n this.gifFramesIndex = 0;\n this.background = HMSVirtualBackgroundTypes.NONE;\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n }\n\n async processVideoFrame(input: HTMLCanvasElement, output: HTMLCanvasElement, skipProcessing?: boolean) {\n if (!input || !output) {\n throw new Error('Plugin invalid input/output');\n }\n this.input = input;\n output.width = input.width;\n output.height = input.height;\n this.outputCanvas = output;\n this.outputCtx = output.getContext('2d');\n if (skipProcessing && this.prevResults) {\n this.handleResults(this.prevResults);\n return;\n }\n if (this.backgroundType === HMSVirtualBackgroundTypes.NONE) {\n this.outputCtx?.drawImage(input, 0, 0, input.width, input.height);\n return;\n }\n await this.segmentation.send({ image: input });\n }\n\n private async setImage(image: HTMLImageElement): Promise<any> {\n image.crossOrigin = 'anonymous';\n return new Promise((resolve, reject) => {\n image.onload = () => resolve(image);\n image.onerror = reject;\n });\n }\n\n private handleResults = (results: MediaPipeResults) => {\n if (!this.outputCanvas || !this.outputCtx) {\n return;\n }\n this.outputCtx.save();\n this.outputCtx.clearRect(0, 0, this.outputCanvas.width, this.outputCanvas.height);\n switch (this.backgroundType) {\n case HMSVirtualBackgroundTypes.IMAGE:\n case HMSVirtualBackgroundTypes.CANVAS:\n case HMSVirtualBackgroundTypes.VIDEO:\n this.renderBackground(results, this.background as HMSBackgroundInput);\n break;\n case HMSVirtualBackgroundTypes.GIF:\n this.renderGIF(results);\n break;\n case HMSVirtualBackgroundTypes.BLUR:\n this.renderBlur(results);\n break;\n }\n this.outputCtx.restore();\n this.prevResults = results;\n };\n\n private loadGIF(url: string): Promise<any> {\n return fetch(url)\n .then(resp => resp.arrayBuffer())\n .then(buff => parseGIF(buff))\n .then(gif => {\n return decompressFrames(gif, true);\n });\n }\n\n private log(...data: any[]) {\n console.debug(this.TAG, ...data);\n }\n\n private renderBackground = (results: MediaPipeResults, background: HMSBackgroundInput) => {\n if (\n !this.input ||\n !this.outputCanvas ||\n !this.outputCtx ||\n this.backgroundType === HMSVirtualBackgroundTypes.NONE ||\n this.backgroundType === HMSVirtualBackgroundTypes.BLUR\n ) {\n return;\n }\n this.outputCtx.filter = 'none';\n this.outputCtx.imageSmoothingEnabled = true;\n this.outputCtx.imageSmoothingQuality = 'high';\n // Only overwrite existing pixels.\n this.outputCtx.globalCompositeOperation = 'source-out';\n const bgWidth = background instanceof HTMLVideoElement ? background.videoWidth : background.width;\n const bgHeight = background instanceof HTMLVideoElement ? background.videoHeight : background.height;\n\n this.outputCtx.drawImage(\n background,\n 0,\n 0,\n bgWidth,\n bgHeight,\n 0,\n 0,\n this.outputCanvas.width,\n this.outputCanvas.height,\n );\n this.outputCtx.globalCompositeOperation = 'destination-out';\n this.outputCtx.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n // Only overwrite missing pixels.\n this.outputCtx.globalCompositeOperation = 'destination-atop';\n this.outputCtx.drawImage(this.input, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n };\n\n private renderBlur(results: MediaPipeResults) {\n if (!this.outputCanvas || !this.outputCtx || this.backgroundType !== HMSVirtualBackgroundTypes.BLUR) {\n return;\n }\n this.outputCtx!.filter = 'none';\n this.outputCtx!.globalCompositeOperation = 'source-out';\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.globalCompositeOperation = 'destination-atop';\n this.outputCtx?.drawImage(results.segmentationMask, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n this.outputCtx!.filter = `blur(${Math.floor(this.outputCanvas.width / 160) * 5}px)`;\n this.outputCtx?.drawImage(results.image, 0, 0, this.outputCanvas.width, this.outputCanvas.height);\n }\n\n private renderGIF(results: MediaPipeResults) {\n if (\n !this.outputCanvas ||\n !this.outputCtx ||\n !this.tempGifContext ||\n this.backgroundType !== HMSVirtualBackgroundTypes.GIF\n ) {\n return;\n }\n if (this.gifFrameImageData == null) {\n const dims = this.gifFrames[this.gifFramesIndex].dims;\n this.tempGifCanvas.width = dims.width;\n this.tempGifCanvas.height = dims.height;\n this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);\n }\n // set the patch data as an override\n this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);\n this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);\n this.gifFramesIndex = (this.gifFramesIndex + 1) % this.gifFrames.length;\n this.renderBackground(results, this.tempGifCanvas);\n }\n}\n\nexport { HMSVirtualBackgroundTypes };\n", "export type HMSBackgroundInput = HTMLImageElement | HTMLVideoElement | HTMLCanvasElement;\nexport enum HMSVirtualBackgroundTypes {\n BLUR = 'blur',\n NONE = 'none',\n GIF = 'gif',\n IMAGE = 'image',\n VIDEO = 'video',\n CANVAS = 'canvas',\n}\n\nexport type HMSVirtualBackground =\n | string\n | HMSVirtualBackgroundTypes.BLUR\n | HMSVirtualBackgroundTypes.NONE\n | HMSBackgroundInput;\n", "import { tsvb } from 'effects-sdk';\nimport { HMSMediaStreamPlugin } from '@100mslive/hms-video-store';\nimport { EFFECTS_SDK_ASSETS } from './constants';\nimport { HMSVirtualBackgroundTypes } from './interfaces';\n\nexport type HMSEffectsBackground = string | MediaStream | MediaStreamTrack | HTMLVideoElement;\n\nexport class HMSEffectsPlugin implements HMSMediaStreamPlugin {\n private effects: tsvb;\n // Ranges from 0 to 1, inclusive\n private blurAmount = 0;\n private background: HMSEffectsBackground = HMSVirtualBackgroundTypes.NONE;\n private backgroundType = HMSVirtualBackgroundTypes.NONE;\n private preset: 'balanced' | 'quality' = 'balanced';\n private initialised = false;\n private intervalId: NodeJS.Timer | null = null;\n private onInit;\n private onResolutionChangeCallback?: (width: number, height: number) => void;\n private canvas: HTMLCanvasElement;\n\n constructor(effectsSDKKey: string, onInit?: () => void) {\n this.effects = new tsvb(effectsSDKKey);\n this.onInit = onInit;\n this.effects.config({\n sdk_url: EFFECTS_SDK_ASSETS,\n models: {\n colorcorrector: '',\n facedetector: '',\n lowlighter: '',\n },\n wasmPaths: {\n 'ort-wasm.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm.wasm`,\n 'ort-wasm-simd.wasm': `${EFFECTS_SDK_ASSETS}ort-wasm-simd.wasm`,\n },\n });\n this.canvas = document.createElement('canvas');\n this.effects.onError(err => {\n // currently logging info type messages as well\n if (!err.type || err.type === 'error') {\n console.error('[HMSEffectsPlugin]', err);\n }\n });\n this.effects.cache();\n this.effects.onReady = () => {\n if (this.effects) {\n this.initialised = true;\n this.onInit?.();\n this.effects.run();\n this.effects.setBackgroundFitMode('fill');\n this.effects.setSegmentationPreset(this.preset);\n this.applyEffect();\n }\n };\n }\n\n getName(): string {\n return 'HMSEffects';\n }\n\n private executeAfterInit(callback: () => void) {\n if (this.initialised) {\n callback();\n }\n\n if (this.intervalId !== null) {\n clearInterval(this.intervalId);\n }\n this.intervalId = setInterval(() => {\n if (this.initialised) {\n clearInterval(this.intervalId!);\n callback();\n }\n }, 100);\n }\n\n removeBlur() {\n this.blurAmount = 0;\n this.executeAfterInit(() => {\n this.effects.clearBlur();\n });\n }\n\n removeBackground() {\n this.background = '';\n this.executeAfterInit(() => {\n this.effects.clearBackground();\n });\n }\n\n /**\n * @param blur ranges between 0 and 1\n */\n setBlur(blur: number) {\n this.blurAmount = blur;\n this.backgroundType = HMSVirtualBackgroundTypes.BLUR;\n this.removeBackground();\n this.executeAfterInit(() => {\n this.effects.setBlur(this.blurAmount);\n });\n }\n\n /**\n * @param preset can be 'quality' or 'balanced'. The 'quality' preset has better quality but higher CPU usage than 'balanced'\n */\n async setPreset(preset: 'quality' | 'balanced') {\n this.preset = preset;\n return new Promise((resolve, reject) => {\n this.executeAfterInit(() => {\n this.effects.setSegmentationPreset(this.preset).then(resolve).catch(reject);\n });\n });\n }\n\n onResolutionChange(callback: (width: number, height: number) => void) {\n this.onResolutionChangeCallback = callback;\n }\n\n getPreset() {\n return this.preset;\n }\n\n removeEffects() {\n this.backgroundType = HMSVirtualBackgroundTypes.NONE;\n this.removeBackground();\n this.removeBlur();\n }\n\n setBackground(url: HMSEffectsBackground) {\n this.background = url;\n this.backgroundType = HMSVirtualBackgroundTypes.IMAGE;\n this.removeBlur();\n this.executeAfterInit(() => {\n this.effects.setBackground(this.background);\n });\n }\n\n getBlurAmount() {\n return this.blurAmount;\n }\n\n getBackground() {\n return this.background || this.backgroundType;\n }\n\n private updateCanvas(stream: MediaStream) {\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.canvas.width = width!;\n this.canvas.height = height!;\n this.effects.useStream(stream);\n this.effects.toCanvas(this.canvas);\n }\n\n apply(stream: MediaStream): MediaStream {\n this.effects.clear();\n this.applyEffect();\n this.effects.onChangeInputResolution(() => {\n this.updateCanvas(stream);\n const { height, width } = stream.getVideoTracks()[0].getSettings();\n this.onResolutionChangeCallback?.(width!, height!);\n });\n this.updateCanvas(stream);\n return this.canvas.captureStream(30) || stream;\n }\n\n stop() {\n this.removeEffects();\n this.executeAfterInit(() => {\n this.effects.stop();\n });\n }\n\n private applyEffect() {\n if (this.blurAmount) {\n this.setBlur(this.blurAmount);\n } else if (this.background) {\n this.setBackground(this.background);\n }\n }\n}\n", "export const EFFECTS_SDK_ASSETS = 'https://assets.100ms.live/effectsdk/3.4.3/';\n"],
5
5
  "mappings": "2QAAA,IAAAA,EAAAC,EAAA,CAAAC,GAAAC,IAAA,CAAAA,EAAA,SACE,QAAW,UACX,QAAW,MACX,KAAQ,oCACR,OAAU,QACV,OAAU,oBACV,KAAQ,oBACR,QAAW,kBACX,cAAiB,CACf,IAAK,CACH,YAAe,CACb,yBACF,EACA,iBAAoB,CAClB,8BACF,CACF,CACF,EACA,QAAW,CACT,IAAK,CACH,OAAU,sBACV,QAAW,sBACX,QAAW,sBACX,MAAS,mBACX,EACA,gBAAiB,CACf,OAAU,4BACV,QAAW,4BACX,QAAW,4BACX,MAAS,yBACX,EACA,qBAAsB,CACpB,OAAU,iCACV,QAAW,iCACX,QAAW,iCACX,MAAS,8BACX,CACF,EACA,WAAc,CACZ,KAAQ,MACR,IAAO,4CACP,UAAa,iCACf,EACA,MAAS,CACP,OACA,aACA,YACF,EACA,QAAW,CACT,MAAS,uCACT,IAAO,yBACP,aAAc,2BACd,MAAS,sCACT,MAAS,SACT,cAAe,uBACf,KAAQ,wCACR,KAAQ,8BACR,WAAY,kBACZ,QAAW,aACX,KAAQ,aACR,QAAW,mBACX,OAAU,8BACZ,EACA,iBAAoB,CAClB,6BAA8B,SAChC,EACA,gBAAmB,CACjB,6BAA8B,SAChC,EACA,aAAgB,CACd,iCAAkC,kBAClC,uCAAwC,SACxC,iCAAkC,SAClC,6BAA8B,UAC9B,wBAAyB,UACzB,sCAAuC,SACvC,0BAA2B,SAC3B,cAAe,QACf,YAAa,SACb,aAAc,QAChB,EACA,aAAgB,CACd,YACA,iBACA,cACA,mBACA,kBACA,mBACF,EACA,QAAW,0CACb,ICzFA,OAAS,oBAAAC,EAAkB,YAAAC,MAAgB,YAC3C,OAEE,6BAAAC,EAEA,sBAAAC,MACK,6BACP,MAAO,iCCRP,IAAMC,EAAM,IACNC,EAAW,qBAAqBD,EAAI,IAAI,OACxCE,EAAM,cACNC,EAAiB,mBACjBC,EAAsB,wBACtBC,EAAkB,8CAElBC,EAAcC,GACX,IAAI,QAAQ,SAAUC,EAASC,EAAQ,CAC5C,IAAMC,EAAI,SAAS,cAAc,QAAQ,EACzCA,EAAE,IAAMH,EACRG,EAAE,OAASF,EACXE,EAAE,QAAUD,EACZ,SAAS,KAAK,YAAYC,CAAC,CAC7B,CAAC,EAGGC,EAAkB,IAAYC,EAAA,wBAClC,IAAIC,EACAC,EAAO,GAAGb,CAAQ,IAAIG,CAAmB,GAC7C,MAAME,EAAWQ,CAAI,EACrB,GAAI,CAEFD,EAAS,MAAM,uBAAuB,CACxC,OAAQ,GACN,QAAQ,KAAK,wEAAwE,EACrFC,EAAO,GAAGb,CAAQ,IAAIE,CAAc,GACpC,MAAMG,EAAWQ,CAAI,EAErBD,EAAS,MAAM,mBAAmB,CACpC,CACA,OAAOA,CACT,GAEME,EAAa,IAAYH,EAAA,wBAC7B,IAAMI,EAAY,GAAGf,CAAQ,IAAII,CAAe,GAC1C,CAACQ,EAAQI,CAAa,EAAI,MAAM,QAAQ,IAAI,CAACN,EAAgB,EAAG,MAAMK,CAAS,CAAC,CAAC,EAEjFE,EAAQ,MAAMD,EAAc,YAAY,EACxCE,EAAoBN,EAAO,4BAA4B,EAC7D,OAAAA,EAAO,OAAO,IAAI,IAAI,WAAWK,CAAK,EAAGC,CAAiB,EAC1DN,EAAO,WAAWK,EAAM,UAAU,EAElC,QAAQ,MAAMhB,EAAK,uBAAwBW,EAAO,sBAAsB,CAAC,EACzE,QAAQ,MAAMX,EAAK,gBAAiBW,EAAO,gBAAgB,CAAC,EAC5D,QAAQ,MAAMX,EAAK,eAAgBW,EAAO,eAAe,CAAC,EAC1D,QAAQ,MAAMX,EAAK,kBAAmBW,EAAO,sBAAsB,CAAC,EAE7DA,CACT,GDtCA,IAAMO,EAAM,cACNC,EAAgB,GAChBC,EAAM,IAENC,GAA6B,IAC7BC,GAA6B,IAC7BC,GAA8B,IAC9BC,GAA8B,IAEvBC,EAAN,KAA2D,CAkChE,YAAYC,EAAoBC,EAAmB,GAAO,CA3B1D,oBAAiB,OA4Bf,KAAK,WAAaD,EAClB,KAAK,iBAAmBC,EAExB,KAAK,gBAAkB,KACvB,KAAK,gBAAkB,KAEvB,KAAK,gBAAkB,IACvB,KAAK,iBAAmB,IACxB,KAAK,oBAAsB,GAC3B,KAAK,UAAY,OACjB,KAAK,gBAAkB,GACvB,KAAK,OAAS,KACd,KAAK,UAAY,yBAEjB,KAAK,UAAY,KACjB,KAAK,MAAQ,KACb,KAAK,OAAS,KACd,KAAK,QAAU,EACf,KAAK,iBAAmB,EAExB,KAAK,qBAAuB,KAAK,gBAAkB,KAAK,iBACxD,KAAK,WAAa,IAAI,UAAU,KAAK,gBAAiB,KAAK,gBAAgB,EAC3E,KAAK,iBAAmB,SAAS,cAAc,QAAQ,EACvD,KAAK,iBAAiB,MAAQ,KAAK,gBACnC,KAAK,iBAAiB,OAAS,KAAK,iBACpC,KAAK,cAAgB,KAAK,iBAAiB,WAAW,IAAI,EAE1D,KAAK,QAAU,CAAC,EAChB,KAAK,UAAY,KACjB,KAAK,eAAiB,EACtB,KAAK,kBAAoB,KACzB,KAAK,cAAgB,SAAS,cAAc,QAAQ,EACpD,KAAK,eAAiB,KAAK,cAAc,WAAW,IAAI,EACxD,KAAK,cAAgB,EACrB,KAAK,iBAAmBA,EAExB,KAAK,IAAIT,EAAK,mCAAmC,EACjD,KAAK,cAAc,KAAK,UAAU,CACpC,CAEM,MAAsB,QAAAU,EAAA,sBACrB,KAAK,gBAOR,MAAM,KAAK,eANX,KAAK,IAAIV,EAAK,4BAA6B,KAAK,MAAM,EACtD,KAAK,gBAAkB,GACvB,KAAK,cAAgBW,EAAW,EAChC,KAAK,OAAS,MAAM,KAAK,eAKvB,KAAK,kBACP,KAAK,kBAAkB,CAE3B,GAKA,aAAuB,CAErB,OACE,UAAU,UAAU,QAAQ,QAAQ,IAAM,IAC1C,UAAU,UAAU,QAAQ,SAAS,IAAM,IAC3C,UAAU,UAAU,QAAQ,KAAK,IAAM,IACvC,UAAU,UAAU,QAAQ,MAAM,IAAM,EAE5C,CAEA,cAAuC,CACrC,IAAMC,EAAgB,CAAC,EACvB,MAAI,CAAC,SAAU,UAAW,MAAO,MAAM,EAAE,KAAKC,GAAS,UAAU,UAAU,QAAQA,CAAK,IAAM,EAAE,EAC9FD,EAAc,YAAc,IAE5BA,EAAc,YAAc,GAC5BA,EAAc,QAAUE,EAA0B,uBAClDF,EAAc,OAAS,8CAGlBA,CACT,CAEA,SAAkB,CAChB,OAAOV,EAAI,IACb,CAEA,eAAoC,CAClC,OAAOa,EAAmB,SAC5B,CAEM,cAAcC,EAAqD,QAAAN,EAAA,sBACvE,GAAIM,IAAS,GACX,GAAIA,IAAS,OACX,KAAK,IAAIhB,EAAK,0BAA2BgB,CAAI,EAC7C,KAAK,WAAa,OAClB,KAAK,eAAiB,OACtB,KAAK,oBAAsB,WAClBA,IAAS,OAClB,KAAK,IAAIhB,EAAK,0BAA2BgB,CAAI,EAC7C,KAAK,WAAa,OAClB,KAAK,eAAiB,OACtB,KAAK,oBAAsB,WAClBA,aAAgB,iBAAkB,CAC3C,KAAK,IAAI,8BAA+BA,CAAI,EAC5C,IAAMC,EAAM,MAAM,KAAK,SAASD,CAAwB,EACxD,GAAI,CAACC,GAAO,CAACA,EAAI,UAAY,CAACA,EAAI,cAChC,MAAM,IAAI,MAAM,yEAAyE,EAEzF,KAAK,oBAAsB,GAC3B,KAAK,gBAAkBA,EACvB,KAAK,eAAiB,OAE1B,SAAWD,aAAgB,iBACzB,KAAK,IAAI,8BAA+BA,CAAI,EAC5C,KAAK,gBAAkBA,EACvB,KAAK,gBAAgB,YAAc,YACnC,KAAK,gBAAgB,MAAQ,GAC7B,KAAK,gBAAgB,KAAO,GAC5B,KAAK,gBAAgB,iBAAmB,IAAYN,EAAA,sBAC9C,KAAK,iBAAmB,OAC1B,MAAM,KAAK,gBAAiB,KAAK,EACjC,KAAK,oBAAsB,GAC3B,KAAK,eAAiB,QAE1B,WAEA,QAAQ,IAAI,2BAA2B,EACvC,KAAK,UAAY,MAAM,KAAK,OAAOM,CAAc,EAC7C,KAAK,WAAa,MAAQ,KAAK,UAAU,OAAS,EACpD,KAAK,eAAiB,MACtB,KAAK,oBAAsB,OAE3B,OAAM,IAAI,MAAM,8EAA8E,MAIlG,OAAM,IAAI,MAAM,8EAA8E,CAElG,GAEA,MAAa,CAlMf,IAAAE,EAAAC,EAmMQ,KAAK,uBACPD,EAAA,KAAK,kBAAL,MAAAA,EAAsB,gBAAgB,QACtCC,EAAA,KAAK,kBAAL,MAAAA,EAAsB,gBAAgB,OAClC,KAAK,iBAAmB,UAC1B,KAAK,gBAAiB,KAAO,GAC7B,KAAK,gBAAkB,OAGvB,KAAK,YACP,KAAK,UAAW,UAAY,eAC5B,KAAK,UAAW,SAAS,EAAG,EAAG,KAAK,OAAQ,MAAO,KAAK,OAAQ,MAAM,GAIxE,KAAK,kBAAoB,KACzB,KAAK,UAAY,KACjB,KAAK,cAAgB,EACrB,KAAK,eAAiB,CACxB,CAEA,kBACEC,EACAC,EACAC,EACsB,CACtB,GAAI,CAACF,GAAS,CAACC,EACb,MAAM,IAAI,MAAM,6BAA6B,EAG/C,KAAK,MAAQD,EACb,KAAK,OAASC,EAEd,IAAME,EAAMF,EAAO,WAAW,IAAI,EAelC,GAdIE,EAAK,OAAO,QAAUH,EAAM,QAC9BG,EAAK,OAAO,MAAQH,EAAM,OAExBG,EAAK,OAAO,SAAWH,EAAM,SAC/BG,EAAK,OAAO,OAASH,EAAM,QAGzB,KAAK,iBAAmB,UAC1B,KAAK,gBAAiB,MAAQA,EAAM,MACpC,KAAK,gBAAiB,OAASA,EAAM,QAGvC,KAAK,UAAYG,EACjB,KAAK,iBAAmBH,EAAM,MAAQA,EAAM,OACxC,KAAK,kBAAoB,EAC3B,MAAM,IAAI,MAAM,4BAA4B,EAG9C,IAAMI,EAAU,IAAYd,EAAA,sBAC1B,MAAM,KAAK,gBAAgBY,CAAc,CAC3C,GAEI,KAAK,aAAe,QAAU,CAAC,KAAK,qBACtC,KAAK,UAAW,yBAA2B,OAC3C,KAAK,UAAW,OAAS,OACzB,KAAK,UAAW,UAAUF,EAAO,EAAG,EAAGA,EAAM,MAAOA,EAAM,MAAM,GAEhEI,EAAQ,CAEZ,CAEc,SAASC,EAAuC,QAAAf,EAAA,sBAC5D,OAAAe,EAAM,YAAc,YACb,IAAI,QAAQ,CAACC,EAASC,IAAW,CACtCF,EAAM,OAAS,IAAMC,EAAQD,CAAK,EAClCA,EAAM,QAAUE,CAClB,CAAC,CACH,GAEQ,OAAOC,EAA2B,CACxC,OAAO,MAAMA,CAAG,EACb,KAAKC,GAAQA,EAAK,YAAY,CAAC,EAC/B,KAAKC,GAAQC,EAASD,CAAI,CAAC,EAC3B,KAAKE,GACGC,EAAiBD,EAAK,EAAI,CAClC,CACL,CAEQ,IAAIE,KAAgBC,EAAa,CACvC,QAAQ,KAAKD,EAAK,GAAGC,CAAI,CAC3B,CAEQ,iBAAkB,CACxB,KAAK,cAAe,UAClB,KAAK,MACL,EACA,EACA,KAAK,MAAO,MACZ,KAAK,MAAO,OACZ,EACA,EACA,KAAK,gBACL,KAAK,gBACP,EAEA,IAAMC,EAAY,KAAK,cAAe,aAAa,EAAG,EAAG,KAAK,gBAAiB,KAAK,gBAAgB,EAE9FC,EAAoB,KAAK,OAAO,sBAAsB,EAAI,EAChE,QAAS,EAAI,EAAG,EAAI,KAAK,qBAAsB,IAC7C,KAAK,OAAO,QAAQA,EAAoB,EAAI,CAAC,EAAID,EAAU,KAAK,EAAI,CAAC,EAAI,IACzE,KAAK,OAAO,QAAQC,EAAoB,EAAI,EAAI,CAAC,EAAID,EAAU,KAAK,EAAI,EAAI,CAAC,EAAI,IACjF,KAAK,OAAO,QAAQC,EAAoB,EAAI,EAAI,CAAC,EAAID,EAAU,KAAK,EAAI,EAAI,CAAC,EAAI,GAErF,CACQ,MAAMd,EAA0B,CACjCA,GACH,KAAK,OAAO,cAAc,EAE5B,IAAMgB,EAAqB,KAAK,OAAO,uBAAuB,EAAI,EAElE,QAAS,EAAI,EAAG,EAAI,KAAK,qBAAsB,IAC7C,GAAI,KAAK,YAAc,OAAQ,CAC7B,IAAM9B,EAAa,KAAK,OAAO,QAAQ8B,EAAqB,EAAI,CAAC,EAC3DC,EAAS,KAAK,OAAO,QAAQD,EAAqB,EAAI,EAAI,CAAC,EAC3DE,EAAQ,KAAK,IAAIhC,EAAY+B,CAAM,EACnCE,EAAgB,KAAK,IAAIjC,EAAagC,CAAK,EAC3CE,EAAY,KAAK,IAAIH,EAASC,CAAK,EAEzC,KAAK,WAAW,KAAK,EAAI,EAAI,CAAC,EAAK,IAAME,GAAcD,EAAgBC,EACzE,SAAW,KAAK,YAAc,yBAA0B,CACtD,IAAMH,EAAS,KAAK,OAAO,QAAQD,EAAqB,CAAC,EACzD,KAAK,WAAW,KAAK,EAAI,EAAI,CAAC,EAAI,IAAMC,CAC1C,CAGF,KAAK,cAAe,aAAa,KAAK,WAAY,EAAG,CAAC,CACxD,CAEQ,gBAAiB,CACvB,KAAK,UAAW,yBAA2B,OAC3C,KAAK,UAAW,OAAS,OAErB,KAAK,oBACP,KAAK,UAAW,OAAS,YAEzB,KAAK,UAAW,OAAS,YAE3B,KAAK,eAAe,EACpB,KAAK,UAAW,yBAA2B,YAC3C,KAAK,UAAW,OAAS,OAEzB,KAAK,UAAW,UAAU,KAAK,MAAQ,EAAG,CAAC,EAGzC,KAAK,kBACL,KAAK,OAAQ,MAAQpC,IACrB,KAAK,OAAQ,OAASE,IACtB,KAAK,OAAQ,MAAQD,IACrB,KAAK,OAAQ,OAASE,IAEtB,KAAK,cAAc,EAIrB,KAAK,wBAAwB,CAC/B,CAEQ,eAAgB,CAQtB,IAAMqC,EAAkB,KAAK,UAAW,aAAa,EAAG,EAAG,KAAK,OAAQ,MAAO,KAAK,OAAQ,MAAM,EAK5FtB,EAAS,KAAK,QAAQ,UAAUsB,CAAe,EAErD,KAAK,UAAW,aAAatB,EAAQ,EAAG,CAAC,CAC3C,CAEQ,gBAAiB,CACvB,KAAK,UAAW,UACd,KAAK,iBACL,EACA,EACA,KAAK,gBACL,KAAK,iBACL,EACA,EACA,KAAK,OAAQ,MACb,KAAK,OAAQ,MACf,CACF,CAEQ,yBAA0B,CAChC,KAAK,UAAW,yBAA2B,mBAC3C,KAAK,UAAW,sBAAwB,GACxC,KAAK,UAAW,sBAAwB,OACpC,KAAK,oBACH,KAAK,iBAAmB,SAAW,KAAK,iBAAmB,MAAQ,KAAK,gBAAiB,YAAc,EACzG,KAAK,qBAAqB,EACjB,KAAK,iBAAmB,QACjC,KAAK,qBAAqB,EACjB,KAAK,iBAAmB,QAC7B,KAAK,cAAgB,KAAK,UAAU,KAAK,cAAc,EAAE,MAAQpB,GACnE,KAAK,iBACD,KAAK,gBAAkB,KAAK,UAAU,SACxC,KAAK,eAAiB,GAExB,KAAK,cAAgB,GAErB,KAAK,gBAEP,KAAK,mBAAmB,GAG1B,KAAK,oBAAoB,CAE7B,CAEc,gBAAgBqB,EAA0B,QAAAZ,EAAA,sBAClD,KAAK,SAGP,KAAK,gBAAgB,EACrB,MAAM,KAAK,MAAMY,CAAc,EAC/B,KAAK,eAAe,EAIxB,GAEQ,sBAAuB,CAC7B,KAAK,QAAQ,KAAK,gBAAiB,KAAK,gBAAiB,WAAY,KAAK,gBAAiB,WAAW,CACxG,CAEQ,sBAAuB,CAC7B,KAAK,QAAQ,KAAK,gBAAiB,KAAK,gBAAiB,MAAO,KAAK,gBAAiB,MAAM,CAC9F,CAEQ,oBAAqB,CAC3B,GAAI,KAAK,mBAAqB,KAAM,CAClC,IAAMsB,EAAO,KAAK,UAAU,KAAK,cAAc,EAAE,KACjD,KAAK,cAAe,MAAQA,EAAK,MACjC,KAAK,cAAe,OAASA,EAAK,OAClC,KAAK,kBAAoB,KAAK,eAAe,gBAAgBA,EAAK,MAAOA,EAAK,MAAM,CACtF,CAEA,KAAK,kBAAkB,KAAK,IAAI,KAAK,UAAU,KAAK,cAAc,EAAE,KAAK,EACzE,KAAK,eAAe,aAAa,KAAK,kBAAmB,EAAG,CAAC,EAE7D,KAAK,QAAQ,KAAK,cAAe,KAAK,kBAAmB,MAAO,KAAK,kBAAmB,MAAM,CAChG,CAEQ,QAAQT,EAAWU,EAAmBC,EAAoB,CAChE,IAAIC,EAAiBC,EAAkBC,EAAcC,EACjDL,EAAYC,EAAa,KAAK,kBAChCC,EAAaF,EACbG,EAAcH,EAAY,KAAK,iBAC/BI,EAAU,EACVC,GAAWJ,EAAaE,GAAe,IAEvCA,EAAcF,EACdC,EAAaD,EAAa,KAAK,iBAC/BI,EAAU,EACVD,GAAWJ,EAAYE,GAAc,GAEvC,KAAK,UAAW,UACdZ,EACAc,EACAC,EACAH,EACAC,EACA,EACA,EACA,KAAK,OAAQ,MACb,KAAK,OAAQ,MACf,CACF,CAEc,qBAAsB,QAAAtC,EAAA,sBAClC,IAAIyC,EAAY,OACZ,KAAK,MAAO,OAAS,IACvBA,EAAY,MACH,KAAK,MAAO,OAAS,IAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,IAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,IAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,KAC9BA,EAAY,OACH,KAAK,MAAO,OAAS,OAC9BA,EAAY,QAGd,KAAK,UAAW,OAAS,QAAQA,CAAS,IAC1C,KAAK,UAAW,UAAU,KAAK,MAAQ,EAAG,EAAG,KAAK,OAAQ,MAAO,KAAK,OAAQ,MAAM,CACtF,GAEQ,mBAAyB,CAC/B,KAAK,QAAQ,UAAY,SAAS,cAAc,QAAQ,EACxD,KAAK,QAAQ,OAAS,KAAK,QAAQ,UAAU,WAAW,IAAI,EAE5D,KAAK,QAAQ,gBAAkB,CAACC,EAAWC,IAClC,KAAK,QAAQ,OAAO,gBAAgBD,EAAGC,CAAC,EAGjD,KAAK,QAAQ,UAAY,CAACC,EAAmBC,EAAU,CAAC,EAAG,GAAI,EAAG,GAAI,EAAG,GAAI,EAAG,GAAI,CAAC,EAAGC,IAAoB,CAC1G,IAAMC,EAAO,KAAK,MAAM,KAAK,KAAKF,EAAQ,MAAM,CAAC,EAE3CG,EAAW,KAAK,MAAMD,EAAO,CAAC,EAC9BE,EAAML,EAAO,KACbM,EAAKN,EAAO,MACZO,EAAKP,EAAO,OAEZF,EAAIQ,EACJP,EAAIQ,EACJxC,EAAS,KAAK,QAAQ,gBAAgB+B,EAAGC,CAAC,EAC1CS,EAAMzC,EAAO,KAEb0C,EAAWP,EAAS,EAAI,EAC9B,QAASQ,EAAI,EAAGA,EAAIX,EAAGW,EAAIA,EAAI,EAC7B,QAASC,EAAI,EAAGA,EAAIb,EAAGa,EAAIA,EAAI,EAAG,CAChC,IAAMC,GAAUF,EAAIZ,EAAIa,GAAK,EAE7B,GAAIN,EAAIO,EAAS,CAAC,IAAM,GAEbD,EAAIb,GAAKY,EAAIX,EAAG,CACzB,IAAMc,EAAKH,EACLI,EAAKH,EAIPI,EAAI,EACNC,EAAI,EACJC,EAAI,EACJC,EAAI,EACN,QAASC,EAAK,EAAGA,EAAKhB,EAAMgB,IAC1B,QAASC,EAAK,EAAGA,EAAKjB,EAAMiB,IAAM,CAChC,IAAMC,EAAMR,EAAKM,EAAKf,EAChBkB,EAAMR,EAAKM,EAAKhB,EACtB,GAAIiB,GAAO,GAAKA,EAAMd,GAAMe,GAAO,GAAKA,EAAMhB,EAAI,CAChD,IAAMiB,GAAUF,EAAMf,EAAKgB,GAAO,EAC5BE,EAAKvB,EAAQkB,EAAKhB,EAAOiB,CAAE,EACjCL,GAAKV,EAAIkB,CAAM,EAAIC,EACnBR,GAAKX,EAAIkB,EAAS,CAAC,EAAIC,EACvBP,GAAKZ,EAAIkB,EAAS,CAAC,EAAIC,EACvBN,GAAKb,EAAIkB,EAAS,CAAC,EAAIC,CACzB,CACF,CAEFhB,EAAII,CAAM,EAAIG,EACdP,EAAII,EAAS,CAAC,EAAII,EAClBR,EAAII,EAAS,CAAC,EAAIK,EAClBT,EAAII,EAAS,CAAC,EAAIM,EAAIT,GAAY,IAAMS,EAC1C,CACF,CAEF,OAAOnD,CACT,CACF,CACF,EE1iBA,OAAsC,sBAAA0D,OAA0B,iCAChE,OAAS,oBAAAC,GAAkB,YAAAC,OAAgB,YAC3C,OAEE,6BAAAC,GAEA,sBAAAC,OACK,6BCPA,IAAKC,OACVA,EAAA,KAAO,OACPA,EAAA,KAAO,OACPA,EAAA,IAAM,MACNA,EAAA,MAAQ,QACRA,EAAA,MAAQ,QACRA,EAAA,OAAS,SANCA,OAAA,IDUL,IAAMC,EAAN,KAA4C,CAgBjD,YAAYC,EAAkCC,EAA2C,CAfzF,KAAQ,IAAM,gBACd,KAAQ,kBACR,KAAQ,sBAqLR,KAAQ,cAAiBC,GAA8B,CACrD,GAAI,GAAC,KAAK,cAAgB,CAAC,KAAK,WAKhC,QAFA,KAAK,UAAU,KAAK,EACpB,KAAK,UAAU,UAAU,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,MAAM,EACxE,KAAK,eAAgB,CAC3B,YACA,aACA,YACE,KAAK,iBAAiBA,EAAS,KAAK,UAAgC,EACpE,MACF,UACE,KAAK,UAAUA,CAAO,EACtB,MACF,WACE,KAAK,WAAWA,CAAO,EACvB,KACJ,CACA,KAAK,UAAU,QAAQ,EACvB,KAAK,YAAcA,EACrB,EAeA,KAAQ,iBAAmB,CAACA,EAA2BF,IAAmC,CACxF,GACE,CAAC,KAAK,OACN,CAAC,KAAK,cACN,CAAC,KAAK,WACN,KAAK,yBACL,KAAK,wBAEL,OAEF,KAAK,UAAU,OAAS,OACxB,KAAK,UAAU,sBAAwB,GACvC,KAAK,UAAU,sBAAwB,OAEvC,KAAK,UAAU,yBAA2B,aAC1C,IAAMG,EAAUH,aAAsB,iBAAmBA,EAAW,WAAaA,EAAW,MACtFI,EAAWJ,aAAsB,iBAAmBA,EAAW,YAAcA,EAAW,OAE9F,KAAK,UAAU,UACbA,EACA,EACA,EACAG,EACAC,EACA,EACA,EACA,KAAK,aAAa,MAClB,KAAK,aAAa,MACpB,EACA,KAAK,UAAU,yBAA2B,kBAC1C,KAAK,UAAU,UAAUF,EAAQ,iBAAkB,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,MAAM,EAE1G,KAAK,UAAU,yBAA2B,mBAC1C,KAAK,UAAU,UAAU,KAAK,MAAO,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,MAAM,CAC9F,EA7OE,KAAK,WAAaF,EAClB,KAAK,eAAiBC,EACtB,KAAK,UAAY,KACjB,KAAK,eAAiB,EACtB,KAAK,kBAAoB,KACzB,KAAK,cAAgB,SAAS,cAAc,QAAQ,EACpD,KAAK,eAAiB,KAAK,cAAc,WAAW,IAAI,EAExD,KAAK,cAAc,KAAK,WAAY,KAAK,cAAc,EACvD,KAAK,IAAI,uCAAuC,CAClD,CAEA,aAAuB,CACrB,OAAO,KAAK,aAAa,EAAE,WAC7B,CAEA,iBAA2B,CACzB,MAAO,WAAY,yBAAyB,SAC9C,CAEA,cAAuC,CACrC,IAAMI,EAAgB,CAAC,EACvB,MAAI,CAAC,SAAU,UAAW,MAAO,OAAQ,QAAQ,EAAE,KAAKC,GAAS,UAAU,UAAU,QAAQA,CAAK,IAAM,EAAE,EACxGD,EAAc,YAAc,IAE5BA,EAAc,YAAc,GAC5BA,EAAc,QAAUE,GAA0B,uBAClDF,EAAc,OAAS,8CAGlBA,CACT,CAEA,SAAkB,CAChB,MAAO,OACT,CAEA,eAAoC,CAClC,OAAOG,GAAmB,SAC5B,CAEM,MAAsB,QAAAC,EAAA,sBACrB,KAAK,eACR,KAAK,aAAe,IAAIC,GAAmB,CACzC,WAAaC,GACJ,mEAAmEA,CAAI,EAElF,CAAC,EACD,KAAK,aAAa,WAAW,CAAE,WAAY,GAAO,eAAgB,CAAE,CAAC,EACrE,KAAK,aAAa,UAAU,KAAK,aAAa,EAElD,GASM,cAAcX,EAAkCC,EAA2C,QAAAQ,EAAA,sBAC/F,GAAI,CAACT,EACH,MAAM,IAAI,MAAM,8EAA8E,EAGhG,OADA,KAAK,YAAc,OACXC,EAAgB,CACtB,WACA,WACE,KAAK,WAAaD,EAClB,KAAK,eAAiBC,EACtB,MACF,YACE,KAAK,IAAI,8BAA+BD,CAAU,EAElD,IAAMY,EAAM,MAAM,KAAK,SAASZ,CAA8B,EAC9D,GAAI,CAACY,GAAO,CAACA,EAAI,UAAY,CAACA,EAAI,cAChC,MAAM,IAAI,MAAM,yEAAyE,EAEzF,KAAK,WAAaA,EAClB,KAAK,uBAEP,MACF,YACE,KAAK,IAAI,8BAA+BZ,CAAU,EAClD,KAAK,sBACL,KAAK,WAAaA,EAClB,KAAK,WAAW,YAAc,YAC9B,KAAK,WAAW,MAAQ,GACxB,KAAK,WAAW,KAAO,GACvB,KAAK,WAAW,YAAc,GAC9B,KAAK,WAAW,iBAAmB,IAAYS,EAAA,sBAC7C,GAAI,KAAK,YAAc,KAAK,sBAAsB,iBAChD,GAAI,CACF,MAAM,KAAK,WAAW,KAAK,EAC3B,KAAK,sBACP,OAASI,EAAG,CACV,KAAK,IAAI,4BAA6Bb,CAAU,CAClD,CAEJ,GACA,MACF,aACE,KAAK,WAAaA,EAClB,KAAK,wBACL,MACF,UAKE,GAJA,KAAK,IAAI,4BAA6BA,CAAU,EAChD,KAAK,sBACL,KAAK,WAAaA,EAClB,KAAK,UAAY,MAAM,KAAK,QAAQ,KAAK,UAAU,EAC/C,KAAK,WAAa,MAAQ,KAAK,UAAU,OAAS,EACpD,KAAK,yBAEL,OAAM,IAAI,MAAM,8EAA8E,EAEhG,MACF,QACE,KAAK,IACH,6EAA6Ec,CAAyB,EACxG,CACJ,CACF,GAEA,eAAgB,CACd,OAAO,KAAK,UACd,CAEA,MAAa,CA3Jf,IAAAC,EA4JQ,KAAK,yBAAqD,KAAK,uBACjEA,EAAA,KAAK,eAAL,MAAAA,EAAmB,SAGrB,KAAK,kBAAoB,KACzB,KAAK,UAAY,KACjB,KAAK,eAAiB,EACtB,KAAK,kBACL,KAAK,qBACP,CAEM,kBAAkBC,EAA0BC,EAA2BC,EAA0B,QAAAT,EAAA,sBAvKzG,IAAAM,EAwKI,GAAI,CAACC,GAAS,CAACC,EACb,MAAM,IAAI,MAAM,6BAA6B,EAO/C,GALA,KAAK,MAAQD,EACbC,EAAO,MAAQD,EAAM,MACrBC,EAAO,OAASD,EAAM,OACtB,KAAK,aAAeC,EACpB,KAAK,UAAYA,EAAO,WAAW,IAAI,EACnCC,GAAkB,KAAK,YAAa,CACtC,KAAK,cAAc,KAAK,WAAW,EACnC,MACF,CACA,GAAI,KAAK,wBAAmD,EAC1DH,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAUC,EAAO,EAAG,EAAGA,EAAM,MAAOA,EAAM,QAC1D,MACF,CACA,MAAM,KAAK,aAAa,KAAK,CAAE,MAAOA,CAAM,CAAC,CAC/C,GAEc,SAASG,EAAuC,QAAAV,EAAA,sBAC5D,OAAAU,EAAM,YAAc,YACb,IAAI,QAAQ,CAACC,EAASC,IAAW,CACtCF,EAAM,OAAS,IAAMC,EAAQD,CAAK,EAClCA,EAAM,QAAUE,CAClB,CAAC,CACH,GAyBQ,QAAQC,EAA2B,CACzC,OAAO,MAAMA,CAAG,EACb,KAAKC,GAAQA,EAAK,YAAY,CAAC,EAC/B,KAAKC,GAAQC,GAASD,CAAI,CAAC,EAC3B,KAAKE,GACGC,GAAiBD,EAAK,EAAI,CAClC,CACL,CAEQ,OAAOE,EAAa,CAC1B,QAAQ,MAAM,KAAK,IAAK,GAAGA,CAAI,CACjC,CAsCQ,WAAW1B,EAA2B,CA3QhD,IAAAa,EAAAc,EAAAC,EA4QQ,CAAC,KAAK,cAAgB,CAAC,KAAK,WAAa,KAAK,0BAGlD,KAAK,UAAW,OAAS,OACzB,KAAK,UAAW,yBAA2B,cAC3Cf,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAUb,EAAQ,MAAO,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,QAC1F,KAAK,UAAW,yBAA2B,oBAC3C2B,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAU3B,EAAQ,iBAAkB,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,QACrG,KAAK,UAAW,OAAS,QAAQ,KAAK,MAAM,KAAK,aAAa,MAAQ,GAAG,EAAI,CAAC,OAC9E4B,EAAA,KAAK,YAAL,MAAAA,EAAgB,UAAU5B,EAAQ,MAAO,EAAG,EAAG,KAAK,aAAa,MAAO,KAAK,aAAa,QAC5F,CAEQ,UAAUA,EAA2B,CAC3C,GACE,GAAC,KAAK,cACN,CAAC,KAAK,WACN,CAAC,KAAK,gBACN,KAAK,wBAIP,IAAI,KAAK,mBAAqB,KAAM,CAClC,IAAM6B,EAAO,KAAK,UAAU,KAAK,cAAc,EAAE,KACjD,KAAK,cAAc,MAAQA,EAAK,MAChC,KAAK,cAAc,OAASA,EAAK,OACjC,KAAK,kBAAoB,KAAK,eAAe,gBAAgBA,EAAK,MAAOA,EAAK,MAAM,CACtF,CAEA,KAAK,kBAAkB,KAAK,IAAI,KAAK,UAAU,KAAK,cAAc,EAAE,KAAK,EACzE,KAAK,eAAe,aAAa,KAAK,kBAAmB,EAAG,CAAC,EAC7D,KAAK,gBAAkB,KAAK,eAAiB,GAAK,KAAK,UAAU,OACjE,KAAK,iBAAiB7B,EAAS,KAAK,aAAa,EACnD,CACF,EE7SA,OAAS,QAAA8B,OAAY,cCAd,IAAMC,EAAqB,6CDO3B,IAAMC,EAAN,KAAuD,CAa5D,YAAYC,EAAuBC,EAAqB,CAVxD,KAAQ,WAAa,EACrB,KAAQ,kBACR,KAAQ,sBACR,KAAQ,OAAiC,WACzC,KAAQ,YAAc,GACtB,KAAQ,WAAkC,KAMxC,KAAK,QAAU,IAAIC,GAAKF,CAAa,EACrC,KAAK,OAASC,EACd,KAAK,QAAQ,OAAO,CAClB,QAASE,EACT,OAAQ,CACN,eAAgB,GAChB,aAAc,GACd,WAAY,EACd,EACA,UAAW,CACT,gBAAiB,GAAGA,CAAkB,gBACtC,qBAAsB,GAAGA,CAAkB,oBAC7C,CACF,CAAC,EACD,KAAK,OAAS,SAAS,cAAc,QAAQ,EAC7C,KAAK,QAAQ,QAAQC,GAAO,EAEtB,CAACA,EAAI,MAAQA,EAAI,OAAS,UAC5B,QAAQ,MAAM,qBAAsBA,CAAG,CAE3C,CAAC,EACD,KAAK,QAAQ,MAAM,EACnB,KAAK,QAAQ,QAAU,IAAM,CA3CjC,IAAAC,EA4CU,KAAK,UACP,KAAK,YAAc,IACnBA,EAAA,KAAK,SAAL,MAAAA,EAAA,WACA,KAAK,QAAQ,IAAI,EACjB,KAAK,QAAQ,qBAAqB,MAAM,EACxC,KAAK,QAAQ,sBAAsB,KAAK,MAAM,EAC9C,KAAK,YAAY,EAErB,CACF,CAEA,SAAkB,CAChB,MAAO,YACT,CAEQ,iBAAiBC,EAAsB,CACzC,KAAK,aACPA,EAAS,EAGP,KAAK,aAAe,MACtB,cAAc,KAAK,UAAU,EAE/B,KAAK,WAAa,YAAY,IAAM,CAC9B,KAAK,cACP,cAAc,KAAK,UAAW,EAC9BA,EAAS,EAEb,EAAG,GAAG,CACR,CAEA,YAAa,CACX,KAAK,WAAa,EAClB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,UAAU,CACzB,CAAC,CACH,CAEA,kBAAmB,CACjB,KAAK,WAAa,GAClB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,gBAAgB,CAC/B,CAAC,CACH,CAKA,QAAQC,EAAc,CACpB,KAAK,WAAaA,EAClB,KAAK,sBACL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,QAAQ,KAAK,UAAU,CACtC,CAAC,CACH,CAKM,UAAUC,EAAgC,QAAAC,EAAA,sBAC9C,YAAK,OAASD,EACP,IAAI,QAAQ,CAACE,EAASC,IAAW,CACtC,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,sBAAsB,KAAK,MAAM,EAAE,KAAKD,CAAO,EAAE,MAAMC,CAAM,CAC5E,CAAC,CACH,CAAC,CACH,GAEA,mBAAmBL,EAAmD,CACpE,KAAK,2BAA6BA,CACpC,CAEA,WAAY,CACV,OAAO,KAAK,MACd,CAEA,eAAgB,CACd,KAAK,sBACL,KAAK,iBAAiB,EACtB,KAAK,WAAW,CAClB,CAEA,cAAcM,EAA2B,CACvC,KAAK,WAAaA,EAClB,KAAK,uBACL,KAAK,WAAW,EAChB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,cAAc,KAAK,UAAU,CAC5C,CAAC,CACH,CAEA,eAAgB,CACd,OAAO,KAAK,UACd,CAEA,eAAgB,CACd,OAAO,KAAK,YAAc,KAAK,cACjC,CAEQ,aAAaC,EAAqB,CACxC,GAAM,CAAE,OAAAC,EAAQ,MAAAC,CAAM,EAAIF,EAAO,eAAe,EAAE,CAAC,EAAE,YAAY,EACjE,KAAK,OAAO,MAAQE,EACpB,KAAK,OAAO,OAASD,EACrB,KAAK,QAAQ,UAAUD,CAAM,EAC7B,KAAK,QAAQ,SAAS,KAAK,MAAM,CACnC,CAEA,MAAMA,EAAkC,CACtC,YAAK,QAAQ,MAAM,EACnB,KAAK,YAAY,EACjB,KAAK,QAAQ,wBAAwB,IAAM,CA3J/C,IAAAR,EA4JM,KAAK,aAAaQ,CAAM,EACxB,GAAM,CAAE,OAAAC,EAAQ,MAAAC,CAAM,EAAIF,EAAO,eAAe,EAAE,CAAC,EAAE,YAAY,GACjER,EAAA,KAAK,6BAAL,MAAAA,EAAA,UAAkCU,EAAQD,EAC5C,CAAC,EACD,KAAK,aAAaD,CAAM,EACjB,KAAK,OAAO,cAAc,EAAE,GAAKA,CAC1C,CAEA,MAAO,CACL,KAAK,cAAc,EACnB,KAAK,iBAAiB,IAAM,CAC1B,KAAK,QAAQ,KAAK,CACpB,CAAC,CACH,CAEQ,aAAc,CAChB,KAAK,WACP,KAAK,QAAQ,KAAK,UAAU,EACnB,KAAK,YACd,KAAK,cAAc,KAAK,UAAU,CAEtC,CACF",
6
6
  "names": ["require_package", "__commonJSMin", "exports", "module", "decompressFrames", "parseGIF", "HMSPluginUnsupportedTypes", "HMSVideoPluginType", "pkg", "BASE_URL", "TAG", "TFLITE_JS_FILE", "TFLITE_SIMD_JS_FILE", "MODEL_FILE_NAME", "loadScript", "src", "resolve", "reject", "s", "loadTFLiteModel", "__async", "tfLite", "path", "loadTFLite", "modelPath", "modelResponse", "model", "modelBufferOffset", "TAG", "DEFAULT_DELAY", "pkg", "minVideoWidthForSharpening", "maxVideoWidthForSharpening", "minVideoHeightForSharpening", "maxVideoHeightForSharpening", "HMSVirtualBackgroundPlugin", "background", "enableSharpening", "__async", "loadTFLite", "browserResult", "value", "HMSPluginUnsupportedTypes", "HMSVideoPluginType", "path", "img", "_a", "_b", "input", "output", "skipProcessing", "ctx", "process", "image", "resolve", "reject", "url", "resp", "buff", "parseGIF", "gif", "decompressFrames", "tag", "data", "imageData", "inputMemoryOffset", "outputMemoryOffset", "person", "shift", "backgroundExp", "personExp", "outputImageData", "dims", "dataWidth", "dataHeight", "inputWidth", "inputHeight", "xoffset", "yoffset", "blurValue", "w", "h", "pixels", "weights", "opaque", "side", "halfSide", "src", "sw", "sh", "dst", "alphaFac", "y", "x", "dstOff", "sy", "sx", "r", "g", "b", "a", "cy", "cx", "scy", "scx", "srcOff", "wt", "SelfieSegmentation", "decompressFrames", "parseGIF", "HMSPluginUnsupportedTypes", "HMSVideoPluginType", "HMSVirtualBackgroundTypes", "HMSVBPlugin", "background", "backgroundType", "results", "bgWidth", "bgHeight", "browserResult", "value", "HMSPluginUnsupportedTypes", "HMSVideoPluginType", "__async", "SelfieSegmentation", "file", "img", "e", "HMSVirtualBackgroundTypes", "_a", "input", "output", "skipProcessing", "image", "resolve", "reject", "url", "resp", "buff", "parseGIF", "gif", "decompressFrames", "data", "_b", "_c", "dims", "tsvb", "EFFECTS_SDK_ASSETS", "HMSEffectsPlugin", "effectsSDKKey", "onInit", "tsvb", "EFFECTS_SDK_ASSETS", "err", "_a", "callback", "blur", "preset", "__async", "resolve", "reject", "url", "stream", "height", "width"]
7
7
  }
package/package.json CHANGED
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "1.13.27",
2
+ "version": "1.13.28",
3
3
  "license": "MIT",
4
4
  "name": "@100mslive/hms-virtual-background",
5
5
  "author": "100ms",
@@ -62,10 +62,10 @@
62
62
  "format": "prettier --write src/**/*.ts"
63
63
  },
64
64
  "peerDependencies": {
65
- "@100mslive/hms-video-store": "0.12.27"
65
+ "@100mslive/hms-video-store": "0.12.28"
66
66
  },
67
67
  "devDependencies": {
68
- "@100mslive/hms-video-store": "0.12.27"
68
+ "@100mslive/hms-video-store": "0.12.28"
69
69
  },
70
70
  "dependencies": {
71
71
  "@mediapipe/selfie_segmentation": "^0.1.1632777926",
@@ -87,5 +87,5 @@
87
87
  "defineTFLite.ts",
88
88
  "importing.test.ts"
89
89
  ],
90
- "gitHead": "bc3bda8727cd8240b6cf3cc6d5ae4e511fdc679b"
90
+ "gitHead": "c4d7367340117e66de386fc7aaa9b5e95abd19ed"
91
91
  }