idmission-web-sdk 2.3.115 → 2.3.116

package/dist/sdk2.esm.js

@@ -7,7 +7,7 @@ import { createPortal } from 'react-dom';
 import { useTranslation, initReactI18next } from 'react-i18next';
 import { Upload } from 'tus-js-client';
 import SparkMD5 from 'spark-md5';
-import { ImageSegmenter, FilesetResolver, ObjectDetector, ImageClassifier } from '@mediapipe/tasks-vision';
+import { ImageSegmenter, FilesetResolver, ObjectDetector, ImageClassifier, FaceDetector } from '@mediapipe/tasks-vision';
 import { useStore, createStore, create } from 'zustand';
 import LanguageDetector from 'i18next-browser-languagedetector';
 import i18n from 'i18next';
@@ -205,7 +205,7 @@ typeof SuppressedError === "function" ? SuppressedError : function (error, suppr
   return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 };
 
-var webSdkVersion = '2.3.115';
+var webSdkVersion = '2.3.116';
 
 function getPlatform() {
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -2348,8 +2348,8 @@ function giveUpAfter(maxTime) {
 
 var DEFAULT_CDN_URL = 'https://websdk-cdn-dev.idmission.com/assets';
 
-var defaultDocumentDetectorModelPath = "".concat(DEFAULT_CDN_URL, "/models/DocumentDetector/DocumentDetector-20251018_121234.tflite");
-var defaultDocumentDetectorModelHash = 'cTLjb4g6H+a+RHnLaYrvVF8050qv/Ox04rMtxFXBDR6Xv8Pv+X//hsaO3K3jNPog';
+var defaultDocumentDetectorModelPath = "".concat(DEFAULT_CDN_URL, "/models/DocumentDetector/DocumentDetector-20250815_115859.tflite");
+var defaultDocumentDetectorModelHash = '64LSaO2ctQsTkGJoZ6Et4v2KH28Fz1ettgkxD+qFg6frxY7ufL9l/lkjw2ljX2cR';
 
 var defaultFocusModelPath = "".concat(DEFAULT_CDN_URL, "/models/Focus/Focus-20241008_102708.tflite");
 var defaultFocusModelHash = 'HTJNLB7QybtPXIIyUI9oNqW40vE5bgSV5V24R1wXLQHknNEVOSyDQDo/QzRgJ8jb';
@@ -2666,8 +2666,8 @@ var documentTypeDisplayNames = {
   singlePage: 'Single page',
   none: 'None'
 };
-var detector = null;
-var detectorSettings = null;
+var detector$1 = null;
+var detectorSettings$1 = null;
 function loadDocumentDetector() {
   return __awaiter(this, arguments, void 0, function (modelAssetPath, scoreThreshold) {
     var _a, _b;
@@ -2680,7 +2680,7 @@ function loadDocumentDetector() {
     return __generator(this, function (_c) {
       switch (_c.label) {
         case 0:
-          if (detector && (detectorSettings === null || detectorSettings === void 0 ? void 0 : detectorSettings.modelAssetPath) === modelAssetPath && (detectorSettings === null || detectorSettings === void 0 ? void 0 : detectorSettings.scoreThreshold) === scoreThreshold) return [2 /*return*/, detector];
+          if (detector$1 && (detectorSettings$1 === null || detectorSettings$1 === void 0 ? void 0 : detectorSettings$1.modelAssetPath) === modelAssetPath && (detectorSettings$1 === null || detectorSettings$1 === void 0 ? void 0 : detectorSettings$1.scoreThreshold) === scoreThreshold) return [2 /*return*/, detector$1];
           closeDocumentDetector();
           return [4 /*yield*/, preloadDocumentDetectorDependencies()];
         case 1:
@@ -2701,20 +2701,20 @@ function loadDocumentDetector() {
             runningMode: 'VIDEO'
           }])];
         case 3:
-          detector = _c.sent();
-          detectorSettings = {
+          detector$1 = _c.sent();
+          detectorSettings$1 = {
             modelAssetPath: modelAssetPath,
             scoreThreshold: scoreThreshold
           };
-          return [2 /*return*/, detector];
+          return [2 /*return*/, detector$1];
       }
     });
   });
 }
 function closeDocumentDetector() {
-  detector === null || detector === void 0 ? void 0 : detector.close();
-  detector = null;
-  detectorSettings = null;
+  detector$1 === null || detector$1 === void 0 ? void 0 : detector$1.close();
+  detector$1 = null;
+  detectorSettings$1 = null;
 }
 function useLoadDocumentDetector(_a) {
   var _b = _a.shouldLoadModels,
@@ -2815,12 +2815,12 @@ function makeDocumentDetectorPrediction(frame) {
   return __awaiter(this, void 0, void 0, function () {
     var startedAt, prediction, time, frameWidth, frameHeight;
     return __generator(this, function (_a) {
-      if (!detector) return [2 /*return*/, null];
+      if (!detector$1) return [2 /*return*/, null];
       startedAt = performance.now();
       // Detectors can throw errors, for example when using custom URLs that
       // contain a model that doesn't provide the expected output.
       try {
-        prediction = detector.detectForVideo(frame, performance.now());
+        prediction = detector$1.detectForVideo(frame, performance.now());
         time = performance.now() - startedAt;
         frameWidth = frame.width;
         frameHeight = frame.height;
@@ -6800,7 +6800,6 @@ function SelfieCaptureFaceKeypoint(_a) {
     scaledHeight = _c.scaledHeight,
     xOffset = _c.xOffset,
     yOffset = _c.yOffset;
-  if (!point) return /*#__PURE__*/React__default.createElement(React__default.Fragment, null);
   var left, top;
   if (horizontal) {
     left = point.x / videoWidth * scaledWidth - xOffset;
@@ -11875,6 +11874,136 @@ var FaceCaptureGuideOverlay = function FaceCaptureGuideOverlay(_a) {
 var templateObject_1$n, templateObject_2$l;
 
 var defaultSelfieCaptureModelLoadTimeoutMs = 45000;
+var detector = null;
+var detectorSettings = null;
+function loadFaceDetector() {
+  return __awaiter(this, arguments, void 0, function (modelAssetPath) {
+    var _a, _b;
+    if (modelAssetPath === void 0) {
+      modelAssetPath = defaultFaceDetectorModelPath;
+    }
+    return __generator(this, function (_c) {
+      switch (_c.label) {
+        case 0:
+          if (detector && (detectorSettings === null || detectorSettings === void 0 ? void 0 : detectorSettings.modelAssetPath) === modelAssetPath) return [2 /*return*/, detector];
+          closeFaceDetector();
+          return [4 /*yield*/, preloadFaceDetectorDependencies()];
+        case 1:
+          _c.sent();
+          if (modelCapabilities.delegate === 'NONE') {
+            throw new Error('No available delegate for face detector.');
+          }
+          _b = (_a = FaceDetector).createFromOptions;
+          return [4 /*yield*/, FilesetResolver.forVisionTasks(visionTasksBasePath)];
+        case 2:
+          return [4 /*yield*/, _b.apply(_a, [_c.sent(), {
+            // canvas: document.createElement('canvas'),
+            baseOptions: {
+              modelAssetPath: modelAssetPath,
+              delegate: modelCapabilities.delegate
+            },
+            runningMode: 'VIDEO'
+          }])];
+        case 3:
+          detector = _c.sent();
+          detectorSettings = {
+            modelAssetPath: modelAssetPath
+          };
+          return [2 /*return*/, detector];
+      }
+    });
+  });
+}
+function closeFaceDetector() {
+  detector === null || detector === void 0 ? void 0 : detector.close();
+  detector = null;
+  detectorSettings = null;
+}
+function useLoadFaceDetector(_a) {
+  var onModelError = _a.onModelError,
+    _b = _a.modelLoadTimeoutMs,
+    modelLoadTimeoutMs = _b === void 0 ? defaultSelfieCaptureModelLoadTimeoutMs : _b,
+    videoRef = _a.videoRef;
+  var _c = useState('not-started'),
+    modelLoadState = _c[0],
+    setModelLoadState = _c[1];
+  var _d = useState(0),
+    modelDownloadProgress = _d[0],
+    setModelDownloadProgress = _d[1];
+  var _e = useState(null),
+    modelWarmingStartedAt = _e[0],
+    setModelWarmingStartedAt = _e[1];
+  var _f = useState(null),
+    modelError = _f[0],
+    setModelError = _f[1];
+  useEffect(function loadModel() {
+    var _this = this;
+    setModelLoadState('downloading');
+    setModelWarmingStartedAt(null);
+    var modelLoadTimeout = setTimeout(function () {
+      setModelError(new Error('Model loading time limit exceeded.'));
+    }, modelLoadTimeoutMs);
+    function handleDownloadProgress(event) {
+      setModelDownloadProgress(progressToPercentage(event.detail));
+    }
+    document.addEventListener('idmission.preloadProgress.faceDetection', handleDownloadProgress);
+    var cancelVideoReady = function cancelVideoReady() {};
+    loadFaceDetector().then(function (model) {
+      return __awaiter(_this, void 0, void 0, function () {
+        var _a, videoReady, cancel, cancelled;
+        return __generator(this, function (_b) {
+          switch (_b.label) {
+            case 0:
+              setModelDownloadProgress(100);
+              clearTimeout(modelLoadTimeout);
+              setModelLoadState('warming');
+              setModelWarmingStartedAt(performance.now());
+              return [4 /*yield*/, testFaceDetectionAgainstKnownImage(model)];
+            case 1:
+              _b.sent();
+              _a = waitForVideoReady(videoRef), videoReady = _a[0], cancel = _a[1];
+              cancelled = false;
+              cancelVideoReady = function cancelVideoReady() {
+                cancelled = true;
+                cancel();
+              };
+              return [4 /*yield*/, videoReady];
+            case 2:
+              _b.sent();
+              if (cancelled) return [2 /*return*/];
+              model.detectForVideo(videoRef.current, performance.now());
+              setModelLoadState('ready');
+              return [2 /*return*/];
+          }
+        });
+      });
+    })["catch"](function (e) {
+      setModelError(e);
+      setModelLoadState('error');
+    })["finally"](function () {
+      clearTimeout(modelLoadTimeout);
+    });
+    return function () {
+      log('unloading face detection model');
+      cancelVideoReady();
+      closeFaceDetector();
+      clearTimeout(modelLoadTimeout);
+      document.removeEventListener('idmission.preloadProgress.faceDetection', handleDownloadProgress);
+    };
+  }, [modelLoadTimeoutMs, videoRef]);
+  useEffect(function handleModelError() {
+    if (modelError) onModelError === null || onModelError === void 0 ? void 0 : onModelError(modelError);
+  }, [modelError, onModelError]);
+  return useMemo(function () {
+    return {
+      ready: modelLoadState === 'ready',
+      modelLoadState: modelLoadState,
+      modelDownloadProgress: modelDownloadProgress,
+      modelWarmingStartedAt: modelWarmingStartedAt,
+      modelError: modelError
+    };
+  }, [modelLoadState, modelDownloadProgress, modelWarmingStartedAt, modelError]);
+}
 var lastFaceDetectionAt = 0;
 var lastFaceDetectionTime = 0;
 function setLastFaceDetectionAt(time) {
@@ -11924,68 +12053,24 @@ function trackFace(face, framesNeeded, frameWidth, frameHeight) {
     if (lastNNosePairs.length > framesNeeded - 1) lastNNosePairs.length = framesNeeded - 1;
   }
 }
-function isFaceDetection(detection) {
-  return detection.boundingBox && detection.categories.some(function (c) {
-    return c.categoryName === 'Primary face' || c.categoryName === 'Secondary face';
+function makeFaceDetectorPrediction(imageData) {
+  if (!detector) return null;
+  var prediction = detector.detectForVideo(imageData, performance.now());
+  var faces = prediction.detections.map(function (d) {
+    return {
+      box: convertBoundingBox(d.boundingBox),
+      keypoints: d.keypoints.map(function (k) {
+        var _a;
+        return _assign(_assign({}, k), {
+          x: k.x * imageData.width,
+          y: k.y * imageData.height,
+          name: (_a = k.label) !== null && _a !== void 0 ? _a : ''
+        });
+      })
+    };
   });
-}
-function isNoseDetection(detection) {
-  return detection.categories.some(function (c) {
-    return c.categoryName === 'Nose';
-  });
-}
-function makeFacePredictionWithDocumentDetector(frame) {
-  return __awaiter(this, void 0, void 0, function () {
-    var prediction, faceDetections, noseDetections, _i, _a, detection, faces, _b, faceDetections_1, d, faceBox, nose, _c, noseDetections_1, n, noseBox, cX, cY;
-    return __generator(this, function (_d) {
-      switch (_d.label) {
-        case 0:
-          return [4 /*yield*/, makeDocumentDetectorPrediction(frame)];
-        case 1:
-          prediction = _d.sent();
-          if (!prediction) return [2 /*return*/, null];
-          faceDetections = [];
-          noseDetections = [];
-          for (_i = 0, _a = prediction.detections; _i < _a.length; _i++) {
-            detection = _a[_i];
-            if (isNoseDetection(detection)) {
-              noseDetections.push(detection);
-            } else if (isFaceDetection(detection)) {
-              faceDetections.push(detection);
-            }
-          }
-          faces = [];
-          for (_b = 0, faceDetections_1 = faceDetections; _b < faceDetections_1.length; _b++) {
-            d = faceDetections_1[_b];
-            faceBox = d.boundingBox;
-            if (!faceBox) continue;
-            nose = null;
-            for (_c = 0, noseDetections_1 = noseDetections; _c < noseDetections_1.length; _c++) {
-              n = noseDetections_1[_c];
-              noseBox = n.boundingBox;
-              if (!noseBox) continue;
-              cX = noseBox.originX + noseBox.width / 2;
-              cY = noseBox.originY + noseBox.height / 2;
-              if (cX >= faceBox.originX && cX <= faceBox.originX + faceBox.width && cY >= faceBox.originY && cY <= faceBox.originY + faceBox.height) {
-                nose = {
-                  x: cX,
-                  y: cY,
-                  name: 'nose'
-                };
-                break;
-              }
-            }
-            faces.push({
-              box: convertBoundingBox(d.boundingBox),
-              keypoints: [null, null, nose, null, null]
-            });
-          }
-          debug('faces', faces);
-          return [2 /*return*/, _assign(_assign({}, prediction), {
-            faces: faces
-          })];
-      }
-    });
+  return _assign(_assign({}, prediction), {
+    faces: faces
   });
 }
 function processFaceDetectorPrediction(_a) {
@@ -12013,9 +12098,13 @@ function processFaceDetectorPrediction(_a) {
     // this represents the edge that the centroid of the face should not cross -- 12.5% of video width
     yCentroidBoundary = _h === void 0 ? 0.125 : _h,
     // this represents the edge that the centroid of the face should not cross -- 12.5% of video height
-    _j = _a.noseTrackingThreshold,
+    _j = _a.foreheadRatio,
     // this represents the edge that the centroid of the face should not cross -- 12.5% of video height
-    noseTrackingThreshold = _j === void 0 ? 0.2 : _j,
+    foreheadRatio = _j === void 0 ? 0.275 : _j,
+    // we found that the bounding box ends at the brow and misses the forehead. this ratio represents how much we should extend the box to include the forehead.
+    _k = _a.noseTrackingThreshold,
+    // we found that the bounding box ends at the brow and misses the forehead. this ratio represents how much we should extend the box to include the forehead.
+    noseTrackingThreshold = _k === void 0 ? 0.2 : _k,
     // this represents the maximum distance that the nose can be from the center of the face box -- 20% of the face box width or height
     minCaptureBrightnessThreshold = _a.minCaptureBrightnessThreshold,
     minCaptureRangeThreshold = _a.minCaptureRangeThreshold,
@@ -12042,10 +12131,10 @@ function processFaceDetectorPrediction(_a) {
     var frameCX = videoWidth / 2;
     var frameCY = videoHeight / 2;
     // calculate head bounding box, with forehead extension
-    // const foreheadSize = face.box.height * foreheadRatio
+    var foreheadSize = face.box.height * foreheadRatio;
     var headXMin = face.box.xMin;
     var headXMax = face.box.xMax;
-    var headYMin = face.box.yMin; // - foreheadSize
+    var headYMin = face.box.yMin - foreheadSize;
     var headYMax = face.box.yMax;
     // calculate head centroids
     var headCX = (headXMin + headXMax) / 2;
@@ -12101,6 +12190,26 @@ function processFaceDetectorPrediction(_a) {
     faceVisibilityTooLow: faceVisibilityTooLow
   };
 }
+function testFaceDetectionAgainstKnownImage(detector) {
+  return new Promise(function (resolve, reject) {
+    var img = new Image();
+    img.crossOrigin = 'anonymous';
+    img.onload = function () {
+      var prediction = detector.detectForVideo(img, performance.now());
+      if (prediction.detections.length > 0) {
+        debug('face detection test result', prediction.detections);
+        resolve(void 0);
+      } else {
+        warn('face detection test failed');
+        reject(new Error('testFaceDetectionAgainstKnownImage failed to predict'));
+      }
+    };
+    img.onerror = function () {
+      return reject(new Error('testFaceDetectionAgainstKnownImage failed to load image'));
+    };
+    img.src = "".concat(DEFAULT_CDN_URL, "/head-test.jpg");
+  });
+}
 
 function detectBrightnessAndContrast(frame, brightnessAverager) {
   var ctx = frame.getContext('2d');
@@ -12252,27 +12361,16 @@ function SelfieGuidanceModelsProvider(_a) {
   var canvasRef = useRef(null);
   var onPredictionHandler = useRef();
   var addToAverage = useRunningAvg(5).addToAverage;
-  var _f = useLoadDocumentDetector({
-      videoRef: videoRef,
+  var _f = useLoadFaceDetector({
       onModelError: onModelError,
-      modelLoadTimeoutMs: modelLoadTimeoutMs
+      modelLoadTimeoutMs: modelLoadTimeoutMs,
+      videoRef: videoRef
     }),
     ready = _f.ready,
     modelLoadState = _f.modelLoadState,
     modelDownloadProgress = _f.modelDownloadProgress,
     modelWarmingStartedAt = _f.modelWarmingStartedAt,
     modelError = _f.modelError;
-  // const {
-  //   ready,
-  //   modelLoadState,
-  //   modelDownloadProgress,
-  //   modelWarmingStartedAt,
-  //   modelError,
-  // } = useLoadFaceDetector({
-  //   onModelError,
-  //   modelLoadTimeoutMs,
-  //   videoRef,
-  // })
   var _g = useFrameLoop(useCallback(function () {
       return __awaiter(_this, void 0, void 0, function () {
         var vw, vh, ctx, thresholdsProvided, brightnessResults, brightness, range, variance, prediction, processed, e_1;
@@ -12286,21 +12384,19 @@ function SelfieGuidanceModelsProvider(_a) {
               ctx = canvasRef.current.getContext('2d');
               canvasRef.current.width = vw;
               canvasRef.current.height = vh;
-              if (!(ctx && videoRef.current.readyState === 4)) return [3 /*break*/, 5];
+              if (!(ctx && videoRef.current.readyState === 4)) return [3 /*break*/, 4];
               ctx.translate(vw, 0);
               ctx.scale(-1, 1);
               ctx.drawImage(videoRef.current, 0, 0, vw, vh);
               _c.label = 1;
             case 1:
-              _c.trys.push([1, 4,, 5]);
+              _c.trys.push([1, 3,, 4]);
               thresholdsProvided = minCaptureBrightnessThreshold !== undefined || minCaptureRangeThreshold !== undefined || minCaptureVarianceThreshold !== undefined;
               brightnessResults = thresholdsProvided ? detectBrightnessAndContrast(canvasRef.current, addToAverage) : undefined;
               brightness = brightnessResults === null || brightnessResults === void 0 ? void 0 : brightnessResults.brightness;
               range = brightnessResults === null || brightnessResults === void 0 ? void 0 : brightnessResults.range;
               variance = brightnessResults === null || brightnessResults === void 0 ? void 0 : brightnessResults.variance;
-              return [4 /*yield*/, makeFacePredictionWithDocumentDetector(canvasRef.current)];
-            case 2:
-              prediction = _c.sent();
+              prediction = makeFaceDetectorPrediction(canvasRef.current);
               processed = processFaceDetectorPrediction({
                 faces: (_a = prediction === null || prediction === void 0 ? void 0 : prediction.faces) !== null && _a !== void 0 ? _a : [],
                 videoWidth: vw,
@@ -12316,15 +12412,15 @@ function SelfieGuidanceModelsProvider(_a) {
               setLastFaceDetectionAt(new Date().getTime());
               // setLastPrediction(processed)
               return [4 /*yield*/, (_b = onPredictionHandler.current) === null || _b === void 0 ? void 0 : _b.call(onPredictionHandler, processed)];
-            case 3:
+            case 2:
               // setLastPrediction(processed)
               _c.sent();
-              return [3 /*break*/, 5];
-            case 4:
+              return [3 /*break*/, 4];
+            case 3:
               e_1 = _c.sent();
               error('caught face detection error', e_1);
-              return [3 /*break*/, 5];
-            case 5:
+              return [3 /*break*/, 4];
+            case 4:
               return [2 /*return*/];
           }
         });
@@ -14861,8 +14957,7 @@ function VideoSignatureCaptureComponent(_a, ref) {
     var rightEdge = videoWidth * (1 - headTrackingBoundaryPercentage);
     var topEdge = videoHeight * headTrackingBoundaryPercentage;
     var bottomEdge = videoHeight * (1 - headTrackingBoundaryPercentage);
-    var nose = face === null || face === void 0 ? void 0 : face.keypoints[2];
-    var nearBoundary = !!face && !!nose && (headTrackingBoundaryType === 'nose' ? nose.x < leftEdge || nose.x > rightEdge || nose.y < topEdge || nose.y > bottomEdge : (face === null || face === void 0 ? void 0 : face.box.xMin) < leftEdge || (face === null || face === void 0 ? void 0 : face.box.xMax) > rightEdge || (face === null || face === void 0 ? void 0 : face.box.yMin) < topEdge || (face === null || face === void 0 ? void 0 : face.box.yMax) > bottomEdge);
+    var nearBoundary = !!face && (headTrackingBoundaryType === 'nose' ? face.keypoints[2].x < leftEdge || face.keypoints[2].x > rightEdge || face.keypoints[2].y < topEdge || face.keypoints[2].y > bottomEdge : (face === null || face === void 0 ? void 0 : face.box.xMin) < leftEdge || (face === null || face === void 0 ? void 0 : face.box.xMax) > rightEdge || (face === null || face === void 0 ? void 0 : face.box.yMin) < topEdge || (face === null || face === void 0 ? void 0 : face.box.yMax) > bottomEdge);
     setLastFace(face);
     setHeadTrackingSatisfied(!!face && !nearBoundary);
     setNumFramesWithoutFaces(face ? 0 : function (n) {