@100mslive/hms-virtual-background 1.3.4 → 1.3.6

package/dist/HMSVirtualBackgroundPlugin.d.ts

@@ -1,11 +1,13 @@
 import '@tensorflow/tfjs-backend-webgl';
-import { HMSVideoPlugin, HMSVideoPluginType } from "@100mslive/hms-video";
+import { HMSVideoPlugin, HMSVideoPluginType } from '@100mslive/hms-video';
 export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     background: string | HTMLImageElement;
     personMaskWidth: number;
     personMaskHeight: number;
     isVirtualBackground: boolean;
-    backgroundImage: HTMLImageElement | undefined;
+    backgroundImage: HTMLImageElement | null;
+    backgroundVideo: HTMLVideoElement | null;
+    backgroundType: string;
     loadModelCalled: boolean;
     blurValue: any;
     tfLite: any;
@@ -22,24 +24,34 @@ export declare class HMSVirtualBackgroundPlugin implements HMSVideoPlugin {
     personMaskCtx: any;
     filters: any;
     enableSharpening?: boolean | false;
+    gifFrames: any;
+    gifFramesIndex: number;
+    gifFrameImageData: any;
+    tempGifCanvas: HTMLCanvasElement;
+    tempGifContext: any;
+    giflocalCount: number;
     constructor(background: string, enableSharpening?: boolean);
     init(): Promise<void>;
     isSupported(): boolean;
     getName(): string;
     getPluginType(): HMSVideoPluginType;
-    setBackground(path?: string | HTMLImageElement): Promise<void>;
+    setBackground(path?: string | HTMLImageElement | HTMLVideoElement): Promise<void>;
     stop(): void;
     processVideoFrame(input: HTMLCanvasElement, output: HTMLCanvasElement, skipProcessing?: boolean): Promise<void> | void;
     private setImage;
+    private setGiF;
     private log;
     private resizeInputData;
     private infer;
     private postProcessing;
-    sharpenFilter(): void;
+    private sharpenFilter;
     private drawPersonMask;
     private drawSegmentedBackground;
     private runSegmentation;
+    private fitVideoToBackground;
     private fitImageToBackground;
+    private fitGifToBackground;
+    private fitData;
     private addBlurToBackground;
-    initSharpenFilter(): any;
+    private initSharpenFilter;
 }

package/dist/hms-virtual-background.cjs.development.js

@@ -3,6 +3,7 @@
 Object.defineProperty(exports, '__esModule', { value: true });
 
 require('@tensorflow/tfjs-backend-webgl');
+var gifuctJs = require('gifuct-js');
 var hmsVideo = require('@100mslive/hms-video');
 
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
@@ -917,8 +918,9 @@ var loadTFLite = /*#__PURE__*/function () {
 }();
 
 var TAG$1 = 'VBProcessor';
+var DEFAULT_DELAY = 33;
 
-var pkg$1 = /*#__PURE__*/require("../package.json");
+var pkg$1 = /*#__PURE__*/require('../package.json');
 
 var minVideoWidthForSharpening = 214;
 var maxVideoWidthForSharpening = 855;
@@ -930,7 +932,11 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
       enableSharpening = false;
     }
 
+    this.backgroundType = 'none';
     this.background = background;
+    this.enableSharpening = enableSharpening;
+    this.backgroundImage = null;
+    this.backgroundVideo = null;
     this.personMaskWidth = 256;
     this.personMaskHeight = 144;
     this.isVirtualBackground = false;
@@ -950,8 +956,14 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     this.personMaskCanvas.height = this.personMaskHeight;
     this.personMaskCtx = this.personMaskCanvas.getContext('2d');
     this.filters = {};
+    this.gifFrames = null;
+    this.gifFramesIndex = 0;
+    this.gifFrameImageData = null;
+    this.tempGifCanvas = document.createElement('canvas');
+    this.tempGifContext = this.tempGifCanvas.getContext('2d');
+    this.giflocalCount = 0;
     this.enableSharpening = enableSharpening;
-    this.log(TAG$1, "Virtual Background plugin created");
+    this.log(TAG$1, 'Virtual Background plugin created');
     this.setBackground(this.background);
   }
 
@@ -968,7 +980,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
                 break;
               }
 
-              this.log(TAG$1, "PREVIOUS LOADED MODEL IS ", this.tfLite);
+              this.log(TAG$1, 'PREVIOUS LOADED MODEL IS ', this.tfLite);
               this.loadModelCalled = true;
               this.tfLitePromise = loadTFLite();
               _context.next = 6;
@@ -1005,7 +1017,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
   _proto.isSupported = function isSupported() {
     //support chrome, firefox, edge TODO: check this
-    return navigator.userAgent.indexOf("Chrome") != -1 || navigator.userAgent.indexOf("Firefox") != -1 || navigator.userAgent.indexOf("Edg") != -1;
+    return navigator.userAgent.indexOf('Chrome') !== -1 || navigator.userAgent.indexOf('Firefox') !== -1 || navigator.userAgent.indexOf('Edg') !== -1;
   };
 
   _proto.getName = function getName() {
@@ -1017,71 +1029,145 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   };
 
   _proto.setBackground = /*#__PURE__*/function () {
-    var _setBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee2(path) {
+    var _setBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee3(path) {
+      var _this = this;
+
       var img;
-      return runtime_1.wrap(function _callee2$(_context2) {
+      return runtime_1.wrap(function _callee3$(_context3) {
         while (1) {
-          switch (_context2.prev = _context2.next) {
+          switch (_context3.prev = _context3.next) {
             case 0:
-              if (!(path !== "")) {
-                _context2.next = 24;
+              if (!(path !== '')) {
+                _context3.next = 50;
                 break;
               }
 
-              if (!(path === "none")) {
-                _context2.next = 7;
+              if (!(path === 'none')) {
+                _context3.next = 8;
                 break;
               }
 
               this.log(TAG$1, 'setting background to :', path);
-              this.background = "none";
+              this.background = 'none';
+              this.backgroundType = 'none';
               this.isVirtualBackground = false;
-              _context2.next = 22;
+              _context3.next = 48;
               break;
 
-            case 7:
-              if (!(path === "blur")) {
-                _context2.next = 13;
+            case 8:
+              if (!(path === 'blur')) {
+                _context3.next = 15;
                 break;
               }
 
               this.log(TAG$1, 'setting background to :', path);
-              this.background = "blur";
+              this.background = 'blur';
+              this.backgroundType = 'blur';
               this.isVirtualBackground = false;
-              _context2.next = 22;
+              _context3.next = 48;
               break;
 
-            case 13:
-              _context2.next = 15;
+            case 15:
+              if (!(path instanceof HTMLImageElement)) {
+                _context3.next = 29;
+                break;
+              }
+
+              this.log('setting background to image', path);
+              _context3.next = 19;
               return this.setImage(path);
 
-            case 15:
-              img = _context2.sent;
+            case 19:
+              img = _context3.sent;
 
               if (!(!img || !img.complete || !img.naturalHeight)) {
-                _context2.next = 20;
+                _context3.next = 24;
                 break;
               }
 
               throw new Error('Invalid image. Provide a valid and successfully loaded HTMLImageElement');
 
-            case 20:
+            case 24:
               this.isVirtualBackground = true;
               this.backgroundImage = img;
+              this.backgroundType = 'image';
 
-            case 22:
-              _context2.next = 25;
+            case 27:
+              _context3.next = 48;
               break;
 
-            case 24:
-              this.log(TAG$1, "Not updating anything using the previous background Settings");
+            case 29:
+              if (!(path instanceof HTMLVideoElement)) {
+                _context3.next = 38;
+                break;
+              }
+
+              this.log('setting background to video', path);
+              this.backgroundVideo = path;
+              this.backgroundVideo.crossOrigin = 'anonymous';
+              this.backgroundVideo.muted = true;
+              this.backgroundVideo.loop = true;
+              this.backgroundVideo.oncanplaythrough = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee2() {
+                return runtime_1.wrap(function _callee2$(_context2) {
+                  while (1) {
+                    switch (_context2.prev = _context2.next) {
+                      case 0:
+                        if (!(_this.backgroundVideo != null)) {
+                          _context2.next = 5;
+                          break;
+                        }
+
+                        _context2.next = 3;
+                        return _this.backgroundVideo.play();
+
+                      case 3:
+                        _this.isVirtualBackground = true;
+                        _this.backgroundType = 'video';
+
+                      case 5:
+                      case "end":
+                        return _context2.stop();
+                    }
+                  }
+                }, _callee2);
+              }));
+              _context3.next = 48;
+              break;
+
+            case 38:
+              console.log('setting gif to background');
+              _context3.next = 41;
+              return this.setGiF(path);
 
-            case 25:
+            case 41:
+              this.gifFrames = _context3.sent;
+
+              if (!(this.gifFrames != null && this.gifFrames.length > 0)) {
+                _context3.next = 47;
+                break;
+              }
+
+              this.backgroundType = 'gif';
+              this.isVirtualBackground = true;
+              _context3.next = 48;
+              break;
+
+            case 47:
+              throw new Error('Invalid background supplied, see the docs to check supported background type');
+
+            case 48:
+              _context3.next = 51;
+              break;
+
+            case 50:
+              throw new Error('Invalid background supplied, see the docs to check supported background type');
+
+            case 51:
             case "end":
-              return _context2.stop();
+              return _context3.stop();
           }
         }
-      }, _callee2, this);
+      }, _callee3, this);
     }));
 
     function setBackground(_x) {
@@ -1093,19 +1179,31 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
   _proto.stop = function stop() {
     if (this.isVirtualBackground) {
-      var _this$backgroundImage;
+      var _this$backgroundImage, _this$backgroundVideo;
 
       (_this$backgroundImage = this.backgroundImage) == null ? void 0 : _this$backgroundImage.removeAttribute('src');
+      (_this$backgroundVideo = this.backgroundVideo) == null ? void 0 : _this$backgroundVideo.removeAttribute('src');
+
+      if (this.backgroundType === 'video') {
+        this.backgroundVideo.loop = false;
+        this.backgroundVideo = null;
+      }
     }
 
     if (this.outputCtx) {
       this.outputCtx.fillStyle = "rgb(0, 0, 0)";
       this.outputCtx.fillRect(0, 0, this.output.width, this.output.height);
-    }
+    } //gif related
+
+
+    this.gifFrameImageData = null;
+    this.gifFrames = null;
+    this.giflocalCount = 0;
+    this.gifFramesIndex = 0;
   };
 
   _proto.processVideoFrame = function processVideoFrame(input, output, skipProcessing) {
-    var _this = this;
+    var _this2 = this;
 
     if (!input || !output) {
       throw new Error('Plugin invalid input/output');
@@ -1123,36 +1221,41 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
       ctx.canvas.height = input.height;
     }
 
+    if (this.backgroundType === 'video') {
+      this.backgroundVideo.width = input.width;
+      this.backgroundVideo.height = input.height;
+    }
+
     this.outputCtx = ctx;
     this.imageAspectRatio = input.width / input.height;
 
     if (this.imageAspectRatio <= 0) {
-      throw new Error("Invalid input width/height");
+      throw new Error('Invalid input width/height');
     }
 
     var process = /*#__PURE__*/function () {
-      var _ref = _asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee3() {
-        return runtime_1.wrap(function _callee3$(_context3) {
+      var _ref2 = _asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee4() {
+        return runtime_1.wrap(function _callee4$(_context4) {
           while (1) {
-            switch (_context3.prev = _context3.next) {
+            switch (_context4.prev = _context4.next) {
               case 0:
-                _context3.next = 2;
-                return _this.runSegmentation(skipProcessing);
+                _context4.next = 2;
+                return _this2.runSegmentation(skipProcessing);
 
               case 2:
               case "end":
-                return _context3.stop();
+                return _context4.stop();
             }
           }
-        }, _callee3);
+        }, _callee4);
       }));
 
       return function process() {
-        return _ref.apply(this, arguments);
+        return _ref2.apply(this, arguments);
       };
     }();
 
-    if (this.background === "none" && !this.isVirtualBackground) {
+    if (this.background === 'none' && !this.isVirtualBackground) {
       this.outputCtx.globalCompositeOperation = 'copy';
       this.outputCtx.filter = 'none';
       this.outputCtx.drawImage(input, 0, 0, input.width, input.height);
@@ -1162,13 +1265,13 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   };
 
   _proto.setImage = /*#__PURE__*/function () {
-    var _setImage = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee4(image) {
-      return runtime_1.wrap(function _callee4$(_context4) {
+    var _setImage = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee5(image) {
+      return runtime_1.wrap(function _callee5$(_context5) {
         while (1) {
-          switch (_context4.prev = _context4.next) {
+          switch (_context5.prev = _context5.next) {
             case 0:
               image.crossOrigin = 'anonymous';
-              return _context4.abrupt("return", new Promise(function (resolve, reject) {
+              return _context5.abrupt("return", new Promise(function (resolve, reject) {
                 image.onload = function () {
                   return resolve(image);
                 };
@@ -1178,10 +1281,10 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
             case 2:
             case "end":
-              return _context4.stop();
+              return _context5.stop();
           }
         }
-      }, _callee4);
+      }, _callee5);
     }));
 
     function setImage(_x2) {
@@ -1191,6 +1294,16 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     return setImage;
   }();
 
+  _proto.setGiF = function setGiF(url) {
+    return fetch(url).then(function (resp) {
+      return resp.arrayBuffer();
+    }).then(function (buff) {
+      return gifuctJs.parseGIF(buff);
+    }).then(function (gif) {
+      return gifuctJs.decompressFrames(gif, true);
+    });
+  };
+
   _proto.log = function log(tag) {
     var _console;
 
@@ -1203,7 +1316,8 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
   _proto.resizeInputData = function resizeInputData() {
     this.personMaskCtx.drawImage(this.input, 0, 0, this.input.width, this.input.height, 0, 0, this.personMaskWidth, this.personMaskHeight);
-    var imageData = this.personMaskCtx.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight);
+    var imageData = this.personMaskCtx.getImageData(0, 0, this.personMaskWidth, this.personMaskHeight); //
+
     var inputMemoryOffset = this.tfLite._getInputMemoryOffset() / 4;
 
     for (var i = 0; i < this.personMaskPixelCount; i++) {
@@ -1289,26 +1403,44 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     this.outputCtx.imageSmoothingQuality = 'high';
 
     if (this.isVirtualBackground) {
-      this.fitImageToBackground();
+      if (this.backgroundType === 'video' && this.backgroundVideo != null && this.backgroundVideo.readyState >= 4) {
+        this.fitVideoToBackground();
+      } else if (this.backgroundType === 'image') {
+        this.fitImageToBackground();
+      } else if (this.backgroundType === 'gif') {
+        if (this.giflocalCount > this.gifFrames[this.gifFramesIndex].delay / DEFAULT_DELAY) {
+          this.gifFramesIndex++;
+
+          if (this.gifFramesIndex >= this.gifFrames.length) {
+            this.gifFramesIndex = 0;
+          }
+
+          this.giflocalCount = 0;
+        } else {
+          this.giflocalCount++;
+        }
+
+        this.fitGifToBackground();
+      }
     } else {
       this.addBlurToBackground();
     }
   };
 
   _proto.runSegmentation = /*#__PURE__*/function () {
-    var _runSegmentation = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee5(skipProcessing) {
-      return runtime_1.wrap(function _callee5$(_context5) {
+    var _runSegmentation = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee6(skipProcessing) {
+      return runtime_1.wrap(function _callee6$(_context6) {
         while (1) {
-          switch (_context5.prev = _context5.next) {
+          switch (_context6.prev = _context6.next) {
             case 0:
               if (!this.tfLite) {
-                _context5.next = 5;
+                _context6.next = 5;
                 break;
               }
 
               // const start = performance.now();
               this.resizeInputData();
-              _context5.next = 4;
+              _context6.next = 4;
               return this.infer(skipProcessing);
 
             case 4:
@@ -1317,10 +1449,10 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
             case 5:
             case "end":
-              return _context5.stop();
+              return _context6.stop();
           }
         }
-      }, _callee5, this);
+      }, _callee6, this);
     }));
 
     function runSegmentation(_x3) {
@@ -1330,30 +1462,52 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
     return runSegmentation;
   }();
 
+  _proto.fitVideoToBackground = function fitVideoToBackground() {
+    this.fitData(this.backgroundVideo, this.backgroundVideo.width, this.backgroundVideo.height);
+  };
+
   _proto.fitImageToBackground = function fitImageToBackground() {
-    var input_width, input_height, x_offset, y_offset;
+    this.fitData(this.backgroundImage, this.backgroundImage.width, this.backgroundImage.height);
+  };
+
+  _proto.fitGifToBackground = function fitGifToBackground() {
+    if (this.gifFrameImageData == null) {
+      var dims = this.gifFrames[this.gifFramesIndex].dims;
+      this.tempGifCanvas.width = dims.width;
+      this.tempGifCanvas.height = dims.height;
+      this.gifFrameImageData = this.tempGifContext.createImageData(dims.width, dims.height);
+    } // set the patch data as an override
+
 
-    if (this.backgroundImage.width / this.backgroundImage.height < this.imageAspectRatio) {
-      input_width = this.backgroundImage.width;
-      input_height = this.backgroundImage.width / this.imageAspectRatio;
-      x_offset = 0;
-      y_offset = (this.backgroundImage.height - input_height) / 2;
+    this.gifFrameImageData.data.set(this.gifFrames[this.gifFramesIndex].patch);
+    this.tempGifContext.putImageData(this.gifFrameImageData, 0, 0);
+    this.fitData(this.tempGifCanvas, this.gifFrameImageData.width, this.gifFrameImageData.height);
+  };
+
+  _proto.fitData = function fitData(data, dataWidth, dataHeight) {
+    var inputWidth, inputHeight, xoffset, yoffset;
+
+    if (dataWidth / dataHeight < this.imageAspectRatio) {
+      inputWidth = dataWidth;
+      inputHeight = dataWidth / this.imageAspectRatio;
+      xoffset = 0;
+      yoffset = (dataHeight - inputHeight) / 2;
     } else {
-      input_height = this.backgroundImage.height;
-      input_width = this.backgroundImage.height * this.imageAspectRatio;
-      y_offset = 0;
-      x_offset = (this.backgroundImage.width - input_width) / 2;
+      inputHeight = dataHeight;
+      inputWidth = dataHeight * this.imageAspectRatio;
+      yoffset = 0;
+      xoffset = (dataWidth - inputWidth) / 2;
     }
 
-    this.outputCtx.drawImage(this.backgroundImage, x_offset, y_offset, input_width, input_height, 0, 0, this.output.width, this.output.height);
+    this.outputCtx.drawImage(data, xoffset, yoffset, inputWidth, inputHeight, 0, 0, this.output.width, this.output.height);
   };
 
   _proto.addBlurToBackground = /*#__PURE__*/function () {
-    var _addBlurToBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee6() {
+    var _addBlurToBackground = /*#__PURE__*/_asyncToGenerator( /*#__PURE__*/runtime_1.mark(function _callee7() {
       var blurValue;
-      return runtime_1.wrap(function _callee6$(_context6) {
+      return runtime_1.wrap(function _callee7$(_context7) {
         while (1) {
-          switch (_context6.prev = _context6.next) {
+          switch (_context7.prev = _context7.next) {
             case 0:
               blurValue = '15px';
               if (this.input.width <= 160) blurValue = '5px';else if (this.input.width <= 320) blurValue = '10px';else if (this.input.width <= 640) blurValue = '15px';else if (this.input.width <= 960) blurValue = '20px';else if (this.input.width <= 1280) blurValue = '25px';else if (this.input.width <= 1920) blurValue = '30px';
@@ -1362,10 +1516,10 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
 
             case 4:
             case "end":
-              return _context6.stop();
+              return _context7.stop();
           }
         }
-      }, _callee6, this);
+      }, _callee7, this);
     }));
 
     function addBlurToBackground() {
@@ -1376,13 +1530,13 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
   }();
 
   _proto.initSharpenFilter = function initSharpenFilter() {
-    var _this2 = this;
+    var _this3 = this;
 
     this.filters.tmpCanvas = document.createElement('canvas');
     this.filters.tmpCtx = this.filters.tmpCanvas.getContext('2d');
 
     this.filters.createImageData = function (w, h) {
-      return _this2.filters.tmpCtx.createImageData(w, h);
+      return _this3.filters.tmpCtx.createImageData(w, h);
     };
 
     this.filters.convolute = function (pixels, weights, opaque) {
@@ -1399,7 +1553,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
       var w = sw;
       var h = sh;
 
-      var output = _this2.filters.createImageData(w, h);
+      var output = _this3.filters.createImageData(w, h);
 
       var dst = output.data; // go through the destination image pixels
 
@@ -1409,7 +1563,7 @@ var HMSVirtualBackgroundPlugin = /*#__PURE__*/function () {
         for (var x = 0; x < w; x = x + 1) {
           var dstOff = (y * w + x) * 4;
 
-          if (src[dstOff + 3] == 0) {
+          if (src[dstOff + 3] === 0) {
             continue;
           } else if (x < w && y < h) {
             var sy = y;