@smileid/web-components 11.4.4 → 11.5.0

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/imageCapture.ts

@@ -0,0 +1,66 @@
+import { JPEG_QUALITY } from '../../../../../domain/constants/src/Constants';
+
+export const captureImageFromVideo = (
+  videoElement: HTMLVideoElement,
+  isReference: boolean = false,
+): string | null => {
+  const canvas = document.createElement('canvas');
+  const ctx = canvas.getContext('2d');
+  if (!ctx) return null;
+
+  const isPortrait = videoElement.videoHeight > videoElement.videoWidth;
+
+  if (isReference) {
+    if (isPortrait) {
+      canvas.width = 480;
+      canvas.height = Math.max(
+        640,
+        (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+      );
+    } else {
+      canvas.width = 640;
+      canvas.height = Math.max(
+        480,
+        (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+      );
+    }
+  } else if (isPortrait) {
+    canvas.width = 240;
+    canvas.height = Math.max(
+      320,
+      (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+    );
+  } else {
+    canvas.width = 320;
+    canvas.height = Math.max(
+      240,
+      (canvas.width * videoElement.videoHeight) / videoElement.videoWidth,
+    );
+  }
+
+  // capture more of the user's head and avoid clipping
+  const zoomOutFactor = 1;
+  const sourceWidth = videoElement.videoWidth * zoomOutFactor;
+  const sourceHeight = videoElement.videoHeight * zoomOutFactor;
+
+  // center the zoomed out area
+  const offsetX = (sourceWidth - videoElement.videoWidth) / 2;
+  const offsetY = (sourceHeight - videoElement.videoHeight) / 2;
+
+  // vertical offset to shift up and capture full head
+  const verticalOffset = 0;
+
+  ctx.drawImage(
+    videoElement,
+    -offsetX,
+    -offsetY - verticalOffset,
+    sourceWidth,
+    sourceHeight,
+    0,
+    0,
+    canvas.width,
+    canvas.height,
+  );
+
+  return canvas.toDataURL('image/jpeg', JPEG_QUALITY);
+};

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/imageQuality.ts

@@ -0,0 +1,151 @@
+/**
+ * Lightweight image quality checks used by the Enhanced SmartSelfie active
+ * liveness flow. Both routines work on small ROIs so they can run inside the
+ * face detection loop without noticeable cost.
+ */
+
+const SAMPLE_STEP = 4; // Sample every 4th pixel — keeps work to ~6% of full ROI.
+
+/**
+ * Module-scoped scratch canvases reused across frames. Allocating a fresh
+ * `<canvas>` per call during the capture loop (~10 fps × 2 routines) creates
+ * meaningful GC churn and re-enters the `willReadFrequently` software path on
+ * every frame. We keep one canvas per routine and only resize when the
+ * computed downsample target changes — assigning `width`/`height` also clears
+ * the bitmap, so no manual clear is required.
+ */
+let luminanceCanvas: HTMLCanvasElement | null = null;
+let luminanceCtx: CanvasRenderingContext2D | null = null;
+let blurCanvas: HTMLCanvasElement | null = null;
+let blurCtx: CanvasRenderingContext2D | null = null;
+
+const getScratchContext = (
+  which: 'luminance' | 'blur',
+  dw: number,
+  dh: number,
+): CanvasRenderingContext2D | null => {
+  let canvas = which === 'luminance' ? luminanceCanvas : blurCanvas;
+  let ctx = which === 'luminance' ? luminanceCtx : blurCtx;
+
+  if (!canvas) {
+    canvas = document.createElement('canvas');
+    ctx = canvas.getContext('2d', { willReadFrequently: true });
+    if (which === 'luminance') {
+      luminanceCanvas = canvas;
+      luminanceCtx = ctx;
+    } else {
+      blurCanvas = canvas;
+      blurCtx = ctx;
+    }
+  }
+  if (!ctx) return null;
+
+  if (canvas.width !== dw) canvas.width = dw;
+  if (canvas.height !== dh) canvas.height = dh;
+  return ctx;
+};
+
+/**
+ * Average BT.601 luma (0–255) over a region of the video frame.
+ *
+ * @param video        live video element
+ * @param region       optional normalised ROI (defaults to centre 60%)
+ */
+export const calculateLuminance = (
+  video: HTMLVideoElement,
+  region?: { x: number; y: number; width: number; height: number },
+): number => {
+  if (!video || video.videoWidth === 0 || video.videoHeight === 0) return 0;
+
+  const w = video.videoWidth;
+  const h = video.videoHeight;
+  const r = region ?? { x: 0.2, y: 0.2, width: 0.6, height: 0.6 };
+
+  const sx = Math.max(0, Math.floor(r.x * w));
+  const sy = Math.max(0, Math.floor(r.y * h));
+  const sw = Math.min(w - sx, Math.floor(r.width * w));
+  const sh = Math.min(h - sy, Math.floor(r.height * h));
+  if (sw <= 0 || sh <= 0) return 0;
+
+  // Downsample heavily — 64x64 max — to keep this near-free.
+  const targetSize = 64;
+  const scale = Math.min(1, targetSize / Math.max(sw, sh));
+  const dw = Math.max(1, Math.round(sw * scale));
+  const dh = Math.max(1, Math.round(sh * scale));
+
+  const ctx = getScratchContext('luminance', dw, dh);
+  if (!ctx) return 0;
+
+  ctx.drawImage(video, sx, sy, sw, sh, 0, 0, dw, dh);
+  const { data } = ctx.getImageData(0, 0, dw, dh);
+
+  let total = 0;
+  let count = 0;
+  for (let i = 0; i < data.length; i += 4 * SAMPLE_STEP) {
+    // BT.601 luma — same approximation OpenCV uses for cvtColor RGB→GRAY.
+    total += 0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2];
+    count += 1;
+  }
+  return count === 0 ? 0 : total / count;
+};
+
+/**
+ * Estimate sharpness as the variance of a 3x3 Laplacian filter applied to the
+ * grayscaled ROI. Higher = sharper. Empirically values < ~30 indicate
+ * meaningful motion blur for a downsampled webcam frame.
+ */
+export const calculateBlurScore = (
+  video: HTMLVideoElement,
+  region?: { x: number; y: number; width: number; height: number },
+): number => {
+  if (!video || video.videoWidth === 0 || video.videoHeight === 0) return 0;
+
+  const w = video.videoWidth;
+  const h = video.videoHeight;
+  const r = region ?? { x: 0.25, y: 0.25, width: 0.5, height: 0.5 };
+
+  const sx = Math.max(0, Math.floor(r.x * w));
+  const sy = Math.max(0, Math.floor(r.y * h));
+  const sw = Math.min(w - sx, Math.floor(r.width * w));
+  const sh = Math.min(h - sy, Math.floor(r.height * h));
+  if (sw <= 0 || sh <= 0) return 0;
+
+  const targetSize = 96;
+  const scale = Math.min(1, targetSize / Math.max(sw, sh));
+  const dw = Math.max(3, Math.round(sw * scale));
+  const dh = Math.max(3, Math.round(sh * scale));
+
+  const ctx = getScratchContext('blur', dw, dh);
+  if (!ctx) return 0;
+
+  ctx.drawImage(video, sx, sy, sw, sh, 0, 0, dw, dh);
+  const { data } = ctx.getImageData(0, 0, dw, dh);
+
+  // Convert to grayscale once.
+  const gray = new Float32Array(dw * dh);
+  for (let i = 0; i < gray.length; i += 1) {
+    const o = i * 4;
+    gray[i] = 0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2];
+  }
+
+  // Variance of 3x3 Laplacian (skip 1-px border).
+  let sum = 0;
+  let sumSq = 0;
+  let n = 0;
+  for (let y = 1; y < dh - 1; y += 1) {
+    for (let x = 1; x < dw - 1; x += 1) {
+      const i = y * dw + x;
+      const lap =
+        4 * gray[i] - gray[i - 1] - gray[i + 1] - gray[i - dw] - gray[i + dw];
+      sum += lap;
+      sumSq += lap * lap;
+      n += 1;
+    }
+  }
+  if (n === 0) return 0;
+  const mean = sum / n;
+  return sumSq / n - mean * mean;
+};
+
+export const DEFAULT_LUMINANCE_MIN = 85; // average luma below this = "too dark"
+export const DEFAULT_BLUR_MIN = 80; // Laplacian variance below this = "too blurry"

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/index.ts

@@ -0,0 +1,5 @@
+export * from './faceDetection';
+export * from './canvas';
+export * from './imageCapture';
+export * from './imageQuality';
+export * from './alertMessages';

package/lib/components/selfie/src/enhanced-smartselfie-capture/utils/mediapipeManager.ts

@@ -0,0 +1,215 @@
+import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
+
+const EXCLUDED_GPUS = [
+  'adreno-830',
+  'adreno-8xx',
+  'adreno-9xx',
+  'adreno-840',
+  'adreno-810',
+];
+
+const normalizeGpuText = (value: string): string =>
+  value
+    .toLowerCase()
+    .replace(/\(tm\)|\btm\b/g, '')
+    .replace(/[^a-z0-9]/g, '');
+
+const matchesExcludedGpu = (value: string): boolean => {
+  const normalizedValue = normalizeGpuText(value);
+
+  return EXCLUDED_GPUS.some((gpuPattern) => {
+    const normalizedPattern = normalizeGpuText(gpuPattern);
+
+    if (normalizedPattern.endsWith('xx')) {
+      const familyPrefix = normalizedPattern.slice(0, -2);
+      return new RegExp(`${familyPrefix}\\d{2}`).test(normalizedValue);
+    }
+
+    return normalizedValue.includes(normalizedPattern);
+  });
+};
+
+/**
+ * @description Gets the GPU renderer string using WebGL debug info extension.
+ * @returns {string | null} The GPU renderer string or null if unavailable.
+ */
+const getGpuRenderer = (): string | null => {
+  try {
+    const canvas = document.createElement('canvas');
+    const gl =
+      canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+    if (!gl || !(gl instanceof WebGLRenderingContext)) return null;
+
+    const ext = gl.getExtension('WEBGL_debug_renderer_info');
+    if (!ext) return null;
+
+    return gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) as string | null;
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * @description Checks if the GPU renderer matches any excluded GPU.
+ * @param {string | null} [renderer] Optional GPU renderer string to use. If not provided, it will be fetched via WebGL.
+ * @returns {boolean} True if the GPU is excluded.
+ */
+const isExcludedGpuFromWebGL = (renderer?: string | null): boolean => {
+  const rendererString = (renderer ?? getGpuRenderer())?.toLowerCase() ?? '';
+  if (!rendererString) return false;
+
+  return matchesExcludedGpu(rendererString);
+};
+
+declare global {
+  interface Window {
+    __smileIdentityMediapipe?: {
+      instance: FaceLandmarker | null;
+      loading: Promise<FaceLandmarker> | null;
+      loaded: boolean;
+    };
+  }
+}
+
+/**
+ * @description Reads system architecture hints from User-Agent Client Hints.
+ * @returns {Promise<string | null>} Lower-cased hint string or null when hints are unavailable.
+ */
+const getSystemArchitectureHints = async (): Promise<string | null> => {
+  if (typeof navigator === 'undefined' || !(navigator as any).userAgentData) {
+    return null;
+  }
+
+  try {
+    const hints = await (navigator as any).userAgentData.getHighEntropyValues([
+      'architecture',
+      'model',
+      'platform',
+      'platformVersion',
+      'fullVersionList',
+    ]);
+
+    return JSON.stringify(hints).toLowerCase();
+  } catch (error) {
+    console.warn('UA-CH architecture hints fetch failed.', error);
+    return null;
+  }
+};
+
+/**
+ * @description Determines the MediaPipe delegate based on WebGL renderer info and UA-CH hints.
+ * Uses WebGL renderer as primary detection, UA-CH hints as secondary.
+ * @returns {Promise<'CPU' | 'GPU'>} CPU when excluded GPU is detected; otherwise GPU.
+ */
+const getDelegateFromGpuDetection = async (): Promise<'CPU' | 'GPU'> => {
+  const renderer = getGpuRenderer();
+  const hasWebGlRendererInfo = !!renderer;
+
+  // Primary check: WebGL renderer info (most reliable for GPU detection)
+  if (isExcludedGpuFromWebGL(renderer)) {
+    console.info(`[SmileID] Excluded GPU via WebGL: ${renderer}. Using CPU.`);
+    return 'CPU';
+  }
+
+  // Secondary check: UA-CH hints (may contain GPU info in some browsers)
+  const hintString = await getSystemArchitectureHints();
+  const hasUaHints = !!hintString;
+
+  if (hintString) {
+    const hasExcludedGpuInHints = matchesExcludedGpu(hintString);
+
+    if (hasExcludedGpuInHints) {
+      console.info(`[SmileID] Excluded GPU via UA-CH hints. Using CPU.`);
+      return 'CPU';
+    }
+  }
+
+  if (!hasWebGlRendererInfo && !hasUaHints) {
+    console.info(
+      '[SmileID] No WebGL renderer or UA-CH hints available. Using CPU.',
+    );
+    return 'CPU';
+  }
+
+  // Default to GPU when no exclusion is detected
+  console.info(
+    `[SmileID] No excluded GPU detected. WebGL renderer: ${renderer ?? 'unavailable'}. Using GPU.`,
+  );
+  return 'GPU';
+};
+
+// this was added because devices (mostly older) that do not support FP16 will fail to load the model.
+const hasFP16Support = () => {
+  const canvas = document.createElement('canvas');
+  const gl =
+    canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+  if (!gl) return false;
+
+  const hasHalfFloatExt = (gl as any).getExtension('OES_texture_half_float');
+  const hasHalfFloatLinear = (gl as any).getExtension(
+    'OES_texture_half_float_linear',
+  );
+  const hasColorBufferHalfFloat = (gl as any).getExtension(
+    'EXT_color_buffer_half_float',
+  );
+
+  return !!(hasHalfFloatExt && hasColorBufferHalfFloat && hasHalfFloatLinear);
+};
+
+export const getMediapipeInstance = async (): Promise<FaceLandmarker> => {
+  if (!window.__smileIdentityMediapipe) {
+    window.__smileIdentityMediapipe = {
+      instance: null,
+      loading: null,
+      loaded: false,
+    };
+  }
+
+  const mediapipeGlobal = window.__smileIdentityMediapipe;
+
+  if (mediapipeGlobal.loaded && mediapipeGlobal.instance) {
+    return mediapipeGlobal.instance;
+  }
+
+  if (mediapipeGlobal.loading) {
+    return mediapipeGlobal.loading;
+  }
+
+  mediapipeGlobal.loading = (async () => {
+    try {
+      const vision = await FilesetResolver.forVisionTasks(
+        'https://web-models.smileidentity.com/mediapipe-tasks-vision-wasm',
+      );
+
+      const gpuDelegate = await getDelegateFromGpuDetection();
+      const delegate =
+        gpuDelegate === 'CPU' || !hasFP16Support() ? 'CPU' : 'GPU';
+
+      const faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
+        baseOptions: {
+          modelAssetPath: `https://web-models.smileidentity.com/face_landmarker/face_landmarker.task`,
+          delegate,
+        },
+        outputFaceBlendshapes: true,
+        runningMode: 'VIDEO',
+        numFaces: 2,
+      });
+
+      mediapipeGlobal.instance = faceLandmarker;
+      mediapipeGlobal.loaded = true;
+      mediapipeGlobal.loading = null;
+
+      return faceLandmarker;
+    } catch (error) {
+      mediapipeGlobal.loading = null;
+      throw error;
+    }
+  })();
+
+  return mediapipeGlobal.loading;
+};
+
+export const __testUtils = {
+  matchesExcludedGpu,
+  getDelegateFromGpuDetection,
+};