@smileid/web-components 11.4.5 → 11.6.0

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/imageCapture.ts

@@ -0,0 +1,66 @@
+import { JPEG_QUALITY } from '../../../../../domain/constants/src/Constants';
+
+export const captureImageFromVideo = (
+  videoElement: HTMLVideoElement,
+  isReference: boolean = false,
+): string | null => {
+  const canvas = document.createElement('canvas');
+  const ctx = canvas.getContext('2d');
+  if (!ctx) return null;
+
+  const isPortrait = videoElement.videoHeight > videoElement.videoWidth;
+
+  if (isReference) {
+    if (isPortrait) {
+      canvas.width = 480;
+      canvas.height = Math.max(
+        640,
+        (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+      );
+    } else {
+      canvas.width = 640;
+      canvas.height = Math.max(
+        480,
+        (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+      );
+    }
+  } else if (isPortrait) {
+    canvas.width = 240;
+    canvas.height = Math.max(
+      320,
+      (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+    );
+  } else {
+    canvas.width = 320;
+    canvas.height = Math.max(
+      240,
+      (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+    );
+  }
+
+  // capture more of the user's head and avoid clipping
+  const zoomOutFactor = 1;
+  const sourceWidth = videoElement.videoWidth * zoomOutFactor;
+  const sourceHeight = videoElement.videoHeight * zoomOutFactor;
+
+  // center the zoomed out area
+  const offsetX = (sourceWidth - videoElement.videoWidth) / 2;
+  const offsetY = (sourceHeight - videoElement.videoHeight) / 2;
+
+  // vertical offset to shift up and capture full head
+  const verticalOffset = 0;
+
+  ctx.drawImage(
+    videoElement,
+    -offsetX,
+    -offsetY - verticalOffset,
+    sourceWidth,
+    sourceHeight,
+    0,
+    0,
+    canvas.width,
+    canvas.height,
+  );
+
+  return canvas.toDataURL('image/jpeg', JPEG_QUALITY);
+};

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/imageQuality.ts

@@ -0,0 +1,151 @@
+/**
+ * Lightweight image quality checks used by the Enhanced SmartSelfie active
+ * liveness flow. Both routines work on small ROIs so they can run inside the
+ * face detection loop without noticeable cost.
+ */
+
+const SAMPLE_STEP = 4; // Sample every 4th pixel — keeps work to ~6% of full ROI.
+
+/**
+ * Module-scoped scratch canvases reused across frames. Allocating a fresh
+ * `<canvas>` per call during the capture loop (~10 fps × 2 routines) creates
+ * meaningful GC churn and re-enters the `willReadFrequently` software path on
+ * every frame. We keep one canvas per routine and only resize when the
+ * computed downsample target changes — assigning `width`/`height` also clears
+ * the bitmap, so no manual clear is required.
+ */
+let luminanceCanvas: HTMLCanvasElement | null = null;
+let luminanceCtx: CanvasRenderingContext2D | null = null;
+let blurCanvas: HTMLCanvasElement | null = null;
+let blurCtx: CanvasRenderingContext2D | null = null;
+
+const getScratchContext = (
+  which: 'luminance' | 'blur',
+  dw: number,
+  dh: number,
+): CanvasRenderingContext2D | null => {
+  let canvas = which === 'luminance' ? luminanceCanvas : blurCanvas;
+  let ctx = which === 'luminance' ? luminanceCtx : blurCtx;
+
+  if (!canvas) {
+    canvas = document.createElement('canvas');
+    ctx = canvas.getContext('2d', { willReadFrequently: true });
+    if (which === 'luminance') {
+      luminanceCanvas = canvas;
+      luminanceCtx = ctx;
+    } else {
+      blurCanvas = canvas;
+      blurCtx = ctx;
+    }
+  }
+  if (!ctx) return null;
+
+  if (canvas.width !== dw) canvas.width = dw;
+  if (canvas.height !== dh) canvas.height = dh;
+  return ctx;
+};
+
+/**
+ * Average BT.601 luma (0–255) over a region of the video frame.
+ *
+ * @param video        live video element
+ * @param region       optional normalised ROI (defaults to centre 60%)
+ */
+export const calculateLuminance = (
+  video: HTMLVideoElement,
+  region?: { x: number; y: number; width: number; height: number },
+): number => {
+  if (!video || video.videoWidth === 0 || video.videoHeight === 0) return 0;
+
+  const w = video.videoWidth;
+  const h = video.videoHeight;
+  const r = region ?? { x: 0.2, y: 0.2, width: 0.6, height: 0.6 };
+
+  const sx = Math.max(0, Math.floor(r.x * w));
+  const sy = Math.max(0, Math.floor(r.y * h));
+  const sw = Math.min(w - sx, Math.floor(r.width * w));
+  const sh = Math.min(h - sy, Math.floor(r.height * h));
+  if (sw <= 0 || sh <= 0) return 0;
+
+  // Downsample heavily — 64x64 max — to keep this near-free.
+  const targetSize = 64;
+  const scale = Math.min(1, targetSize / Math.max(sw, sh));
+  const dw = Math.max(1, Math.round(sw * scale));
+  const dh = Math.max(1, Math.round(sh * scale));
+
+  const ctx = getScratchContext('luminance', dw, dh);
+  if (!ctx) return 0;
+
+  ctx.drawImage(video, sx, sy, sw, sh, 0, 0, dw, dh);
+  const { data } = ctx.getImageData(0, 0, dw, dh);
+
+  let total = 0;
+  let count = 0;
+  for (let i = 0; i < data.length; i += 4 * SAMPLE_STEP) {
+    // BT.601 luma — same approximation OpenCV uses for cvtColor RGB→GRAY.
+    total += 0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2];
+    count += 1;
+  }
+  return count === 0 ? 0 : total / count;
+};
+
+/**
+ * Estimate sharpness as the variance of a 3x3 Laplacian filter applied to the
+ * grayscaled ROI. Higher = sharper. Empirically values < ~30 indicate
+ * meaningful motion blur for a downsampled webcam frame.
+ */
+export const calculateBlurScore = (
+  video: HTMLVideoElement,
+  region?: { x: number; y: number; width: number; height: number },
+): number => {
+  if (!video || video.videoWidth === 0 || video.videoHeight === 0) return 0;
+
+  const w = video.videoWidth;
+  const h = video.videoHeight;
+  const r = region ?? { x: 0.25, y: 0.25, width: 0.5, height: 0.5 };
+
+  const sx = Math.max(0, Math.floor(r.x * w));
+  const sy = Math.max(0, Math.floor(r.y * h));
+  const sw = Math.min(w - sx, Math.floor(r.width * w));
+  const sh = Math.min(h - sy, Math.floor(r.height * h));
+  if (sw <= 0 || sh <= 0) return 0;
+
+  const targetSize = 96;
+  const scale = Math.min(1, targetSize / Math.max(sw, sh));
+  const dw = Math.max(3, Math.round(sw * scale));
+  const dh = Math.max(3, Math.round(sh * scale));
+
+  const ctx = getScratchContext('blur', dw, dh);
+  if (!ctx) return 0;
+
+  ctx.drawImage(video, sx, sy, sw, sh, 0, 0, dw, dh);
+  const { data } = ctx.getImageData(0, 0, dw, dh);
+
+  // Convert to grayscale once.
+  const gray = new Float32Array(dw * dh);
+  for (let i = 0; i < gray.length; i += 1) {
+    const o = i * 4;
+    gray[i] = 0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2];
+  }
+
+  // Variance of 3x3 Laplacian (skip 1-px border).
+  let sum = 0;
+  let sumSq = 0;
+  let n = 0;
+  for (let y = 1; y < dh - 1; y += 1) {
+    for (let x = 1; x < dw - 1; x += 1) {
+      const i = y * dw + x;
+      const lap =
+        4 * gray[i] - gray[i - 1] - gray[i + 1] - gray[i - dw] - gray[i + dw];
+      sum += lap;
+      sumSq += lap * lap;
+      n += 1;
+    }
+  }
+  if (n === 0) return 0;
+  const mean = sum / n;
+  return sumSq / n - mean * mean;
+};
+
+export const DEFAULT_LUMINANCE_MIN = 85; // average luma below this = "too dark"
+export const DEFAULT_BLUR_MIN = 80; // Laplacian variance below this = "too blurry"

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/index.ts

@@ -0,0 +1,5 @@
+export * from './faceDetection';
+export * from './canvas';
+export * from './imageCapture';
+export * from './imageQuality';
+export * from './alertMessages';

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/mediapipeManager.ts

@@ -0,0 +1,215 @@
+import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
+
+const EXCLUDED_GPUS = [
+  'adreno-830',
+  'adreno-8xx',
+  'adreno-9xx',
+  'adreno-840',
+  'adreno-810',
+];
+
+const normalizeGpuText = (value: string): string =>
+  value
+    .toLowerCase()
+    .replace(/\(tm\)|\btm\b/g, '')
+    .replace(/[^a-z0-9]/g, '');
+
+const matchesExcludedGpu = (value: string): boolean => {
+  const normalizedValue = normalizeGpuText(value);
+
+  return EXCLUDED_GPUS.some((gpuPattern) => {
+    const normalizedPattern = normalizeGpuText(gpuPattern);
+
+    if (normalizedPattern.endsWith('xx')) {
+      const familyPrefix = normalizedPattern.slice(0, -2);
+      return new RegExp(`${familyPrefix}\\d{2}`).test(normalizedValue);
+    }
+
+    return normalizedValue.includes(normalizedPattern);
+  });
+};
+
+/**
+ * @description Gets the GPU renderer string using WebGL debug info extension.
+ * @returns {string | null} The GPU renderer string or null if unavailable.
+ */
+const getGpuRenderer = (): string | null => {
+  try {
+    const canvas = document.createElement('canvas');
+    const gl =
+      canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (!gl || !(gl instanceof WebGLRenderingContext)) return null;
+
+    const ext = gl.getExtension('WEBGL_debug_renderer_info');
+    if (!ext) return null;
+
+    return gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) as string | null;
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * @description Checks if the GPU renderer matches any excluded GPU.
+ * @param {string | null} [renderer] Optional GPU renderer string to use. If not provided, it will be fetched via WebGL.
+ * @returns {boolean} True if the GPU is excluded.
+ */
+const isExcludedGpuFromWebGL = (renderer?: string | null): boolean => {
+  const rendererString = (renderer ?? getGpuRenderer())?.toLowerCase() ?? '';
+  if (!rendererString) return false;
+
+  return matchesExcludedGpu(rendererString);
+};
+
+declare global {
+  interface Window {
+    __smileIdentityMediapipe?: {
+      instance: FaceLandmarker | null;
+      loading: Promise<FaceLandmarker> | null;
+      loaded: boolean;
+    };
+  }
+}
+
+/**
+ * @description Reads system architecture hints from User-Agent Client Hints.
+ * @returns {Promise<string | null>} Lower-cased hint string or null when hints are unavailable.
+ */
+const getSystemArchitectureHints = async (): Promise<string | null> => {
+  if (typeof navigator === 'undefined' || !(navigator as any).userAgentData) {
+    return null;
+  }
+
+  try {
+    const hints = await (navigator as any).userAgentData.getHighEntropyValues([
+      'architecture',
+      'model',
+      'platform',
+      'platformVersion',
+      'fullVersionList',
+    ]);
+
+    return JSON.stringify(hints).toLowerCase();
+  } catch (error) {
+    console.warn('UA-CH architecture hints fetch failed.', error);
+    return null;
+  }
+};
+
+/**
+ * @description Determines the MediaPipe delegate based on WebGL renderer info and UA-CH hints.
+ * Uses WebGL renderer as primary detection, UA-CH hints as secondary.
+ * @returns {Promise<'CPU' | 'GPU'>} CPU when excluded GPU is detected; otherwise GPU.
+ */
+const getDelegateFromGpuDetection = async (): Promise<'CPU' | 'GPU'> => {
+  const renderer = getGpuRenderer();
+  const hasWebGlRendererInfo = !!renderer;
+
+  // Primary check: WebGL renderer info (most reliable for GPU detection)
+  if (isExcludedGpuFromWebGL(renderer)) {
+    console.info(`[SmileID] Excluded GPU via WebGL: ${renderer}. Using CPU.`);
+    return 'CPU';
+  }
+
+  // Secondary check: UA-CH hints (may contain GPU info in some browsers)
+  const hintString = await getSystemArchitectureHints();
+  const hasUaHints = !!hintString;
+
+  if (hintString) {
+    const hasExcludedGpuInHints = matchesExcludedGpu(hintString);
+
+    if (hasExcludedGpuInHints) {
+      console.info(`[SmileID] Excluded GPU via UA-CH hints. Using CPU.`);
+      return 'CPU';
+    }
+  }
+
+  if (!hasWebGlRendererInfo && !hasUaHints) {
+    console.info(
+      '[SmileID] No WebGL renderer or UA-CH hints available. Using CPU.',
+    );
+    return 'CPU';
+  }
+
+  // Default to GPU when no exclusion is detected
+  console.info(
+    `[SmileID] No excluded GPU detected. WebGL renderer: ${renderer ?? 'unavailable'}. Using GPU.`,
+  );
+  return 'GPU';
+};
+
+// this was added because devices (mostly older) that do not support FP16 will fail to load the model.
+const hasFP16Support = () => {
+  const canvas = document.createElement('canvas');
+  const gl =
+    canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+  if (!gl) return false;
+
+  const hasHalfFloatExt = (gl as any).getExtension('OES_texture_half_float');
+  const hasHalfFloatLinear = (gl as any).getExtension(
+    'OES_texture_half_float_linear',
+  );
+  const hasColorBufferHalfFloat = (gl as any).getExtension(
+    'EXT_color_buffer_half_float',
+  );
+
+  return !!(hasHalfFloatExt && hasColorBufferHalfFloat && hasHalfFloatLinear);
+};
+
+export const getMediapipeInstance = async (): Promise<FaceLandmarker> => {
+  if (!window.__smileIdentityMediapipe) {
+    window.__smileIdentityMediapipe = {
+      instance: null,
+      loading: null,
+      loaded: false,
+    };
+  }
+
+  const mediapipeGlobal = window.__smileIdentityMediapipe;
+
+  if (mediapipeGlobal.loaded && mediapipeGlobal.instance) {
+    return mediapipeGlobal.instance;
+  }
+
+  if (mediapipeGlobal.loading) {
+    return mediapipeGlobal.loading;
+  }
+
+  mediapipeGlobal.loading = (async () => {
+    try {
+      const vision = await FilesetResolver.forVisionTasks(
+        'https://web-models.smileidentity.com/mediapipe-tasks-vision-wasm',
+      );
+
+      const gpuDelegate = await getDelegateFromGpuDetection();
+      const delegate =
+        gpuDelegate === 'CPU' || !hasFP16Support() ? 'CPU' : 'GPU';
+
+      const faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
+        baseOptions: {
+          modelAssetPath: `https://web-models.smileidentity.com/face_landmarker/face_landmarker.task`,
+          delegate,
+        },
+        outputFaceBlendshapes: true,
+        runningMode: 'VIDEO',
+        numFaces: 2,
+      });
+
+      mediapipeGlobal.instance = faceLandmarker;
+      mediapipeGlobal.loaded = true;
+      mediapipeGlobal.loading = null;
+
+      return faceLandmarker;
+    } catch (error) {
+      mediapipeGlobal.loading = null;
+      throw error;
+    }
+  })();
+
+  return mediapipeGlobal.loading;
+};
+
+export const __testUtils = {
+  matchesExcludedGpu,
+  getDelegateFromGpuDetection,
+};

package/lib/components/selfie/src/selfie-capture-wrapper/SelfieCaptureWrapper.tsx

@@ -6,6 +6,7 @@ import type { FunctionComponent } from 'preact';
 import { getBoolProp } from '../../../../utils/props';
 import { translate, translateHtml } from '../../../../domain/localisation';
 import SmartSelfieCapture from '../smartselfie-capture/SmartSelfieCapture';
+import EnhancedSmartSelfieCapture from '../enhanced-smartselfie-capture/EnhancedSmartSelfieCapture';
 // Legacy web component fallback (used when Mediapipe isn't available)
 import '../selfie-capture/SelfieCapture';
 // Mediapipe loader/manager used by SmartSelfieCapture
@@ -40,6 +41,8 @@ interface Props {
   'show-agent-mode-for-tests'?: string | boolean;
   'hide-attribution'?: string | boolean;
   'disable-image-tests'?: string | boolean;
+  'use-strict-mode'?: string | boolean;
+  'show-back-on-guidelines'?: string | boolean;
   key?: string;
   'start-countdown'?: string | boolean;
   hidden?: string | boolean;
@@ -73,6 +76,7 @@ const SelfieCaptureWrapper: FunctionComponent<Props> = ({
   timeout,
   'start-countdown': startCountdownProp = false,
   'allow-legacy-selfie-fallback': allowLegacySelfieFallbackProp = false,
+  'use-strict-mode': useStrictModeProp = false,
   hidden: hiddenProp = false,
   ...props
 }) => {
@@ -106,7 +110,11 @@ const SelfieCaptureWrapper: FunctionComponent<Props> = ({
 
   const hidden = getBoolProp(hiddenProp);
   const startCountdown = getBoolProp(startCountdownProp);
-  const allowLegacySelfieFallback = getBoolProp(allowLegacySelfieFallbackProp);
+  const useStrictMode = getBoolProp(useStrictModeProp);
+  // Strict mode (Enhanced SmartSelfie) requires Mediapipe head-pose detection,
+  // so the legacy fallback is force-disabled regardless of the partner setting.
+  const allowLegacySelfieFallback =
+    !useStrictMode && getBoolProp(allowLegacySelfieFallbackProp);
 
   // Resolve how long we'll wait for Mediapipe before the hard deadline fires.
   // Precedence:
@@ -463,6 +471,19 @@ const SelfieCaptureWrapper: FunctionComponent<Props> = ({
     return null;
   }
 
+  // Strict mode (Enhanced SmartSelfie) owns its own loading UX. Mount it
+  // immediately — the guidelines screen doesn't need Mediapipe, and by the
+  // time the user reaches the capture screen the background load started by
+  // `useFaceCapture.initializeFaceLandmarker()` will normally have resolved.
+  if (useStrictMode) {
+    return (
+      <>
+        <style>{`:host { display: block; height: 100%; }`}</style>
+        <EnhancedSmartSelfieCapture {...(props as any)} />
+      </>
+    );
+  }
+
   // on retakes, prefer SmartSelfieCapture if Mediapipe is ready
   if (initialSessionCompleted && mediapipeReady && !usingSelfieCapture) {
     return <SmartSelfieCapture {...props} />;
@@ -613,6 +634,8 @@ if (!customElements.get('selfie-capture-wrapper')) {
       'show-agent-mode-for-tests',
       'hide-attribution',
       'disable-image-tests',
+      'use-strict-mode',
+      'show-back-on-guidelines',
       'key',
       'start-countdown',
       'hidden',

package/lib/components/selfie/src/smartselfie-capture/SmartSelfieCapture.tsx

@@ -237,11 +237,11 @@ const SmartSelfieCapture: FunctionComponent<Props> = ({
           font-family: "DM Sans", sans-serif;
           cursor: pointer;
         }
-        
+
         button.btn-primary:hover {
           background-color: #2d2b2a;
         }
-          
+
         button.btn-primary:disabled {
           background-color: #666;
           cursor: not-allowed;

package/lib/components/selfie/src/smartselfie-capture/hooks/useFaceCapture.ts

@@ -336,6 +336,17 @@ export const useFaceCapture = ({
             }, 0);
           }
         }
+
+        // Before the smile zone, resume automatically when face returns to a
+        // valid position — no smile is needed yet.
+        if (
+          isPaused.value &&
+          isCapturing.value &&
+          capturesTaken.value < smileCheckpoint.value &&
+          resumeCaptureRef.current
+        ) {
+          resumeCaptureRef.current();
+        }
       } else {
         // No face detected - reset values
         currentSmileScore.value = 0;
@@ -414,9 +425,7 @@ export const useFaceCapture = ({
       };
 
       window.dispatchEvent(
-        new CustomEvent('selfie-capture.publish', {
-          detail: eventDetail,
-        }),
+        new CustomEvent('selfie-capture.publish', { detail: eventDetail }),
       );
 
       hasFinishedCapture.value = true;
@@ -515,10 +524,9 @@ export const useFaceCapture = ({
     smartCameraWeb?.dispatchEvent(
       new CustomEvent('metadata.selfie-origin', {
         detail: {
-          imageOrigin: {
-            environment: 'back_camera',
-            user: 'front_camera',
-          }[getFacingMode()],
+          imageOrigin: { environment: 'back_camera', user: 'front_camera' }[
+            getFacingMode()
+          ],
         },
       }),
     );

package/lib/components/selfie/src/smartselfie-capture/utils/canvas.ts

@@ -1,8 +1,8 @@
-import { DrawingUtils, FaceLandmarker } from '@mediapipe/tasks-vision';
-
 /**
  * Create a cropped square canvas from video for face detection
  */
+import { DrawingUtils, FaceLandmarker } from '@mediapipe/tasks-vision';
+
 export const createCroppedVideoFrame = (
   videoElement: HTMLVideoElement,
 ): HTMLCanvasElement | null => {
@@ -51,16 +51,14 @@ export const drawFaceMesh = (
   const canvasHeight = canvas.height;
 
   ctx.clearRect(0, 0, canvasWidth, canvasHeight);
-  const drawingUtils = new DrawingUtils(ctx);
 
-  // use this if scaling is needed
-  // const scaleFactor = Math.sqrt(canvasWidth * canvasHeight) / 500;
+  const drawingUtils = new DrawingUtils(ctx);
 
   landmarks.forEach((landmark) => {
     if (!landmark || landmark.length === 0) return;
 
     const outlineColor = 'rgba(162, 155, 254,0.4)';
-    const lineWidth = 2; // Math.max(1, scaleFactor * 2);
+    const lineWidth = 2;
     ctx.strokeStyle = 'rgba(255, 255, 255, 0.6)';
     ctx.lineWidth = lineWidth;
     ctx.lineCap = 'round';

package/lib/components/signature-pad/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smileid/signature-pad",
-  "version": "11.4.5",
+  "version": "11.6.0",
   "private": true,
   "exports": {
     ".": "./index.js"

package/lib/components/smart-camera-web/src/README.md

@@ -193,6 +193,17 @@ After installation and necessary imports:
 
    This approach can also be achieved using other Server to Server libraries.
 
+### Enhanced SmartSelfie
+
+For web-component integrations, Enhanced SmartSelfie is enabled with the
+`use-strict-mode` attribute on the component.
+
+```html
+<smart-camera-web use-strict-mode="true"></smart-camera-web>
+```
+
+- Set `use-strict-mode="true"` to enable strict-mode selfie capture.
+
 ## Compatibility
 
 `SmartCameraWeb` is compatible with most JavaScript frameworks and libraries. For integration with [ReactJS](https://reactjs.org), refer to this [tutorial](https://www.robinwieruch.de/react-web-components) due to React-WebComponents compatibility issues.