@smileid/usesmileid_mlkit_face 12.0.0-beta01 → 12.0.0

package/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 12.0.0
+
+First stable release of the Smile ID v12 SDK. Provides the Android Google ML Kit face detection module for `usesmileid`.
+
 ## 12.0.0-beta01
 
 First public beta release. Android ML Kit face detection for the Smile ID React Native (Expo) SDK v12.

package/README.md

@@ -6,19 +6,13 @@ UseSmileID v12 — Google ML Kit face provider for React Native (Expo Modules).
 
 ## Architecture
 
-The native side runs the entire face-scan pipeline (analyzer + validator
-
-- transitioner + state machine). This package contributes:
-
-* A Kotlin Expo Module that constructs `UseSmileIDFaceSession(FaceDetectorAnalyzer())`
-  from the existing native session.
-* A TypeScript wrapper that bridges the four contract methods
-  (`configure / analyze / reset / close`) and validates the wire payloads
-  at the channel boundary via Zod.
+The native side runs the entire face-scan pipeline (analyzer + validator +
+transitioner + state machine). This package contributes a Kotlin Expo Module
+that constructs `UseSmileIDFaceSession(FaceDetectorAnalyzer())` and a TypeScript
+wrapper that bridges the four contract methods (`configure / analyze / reset /
+close`) with Zod-validated wire payloads.
 
 No ML logic is reimplemented in this package — it's a pure passthrough.
-See `improvements.md` and `react-native-expo/docs/Architecture.md` for
-the design rationale.
 
 ## Installation
 
@@ -58,9 +52,3 @@ await useSmileIDMlkitFace.close();
 ```sh
 pnpm --filter @smileid/usesmileid_mlkit_face test
 ```
-
-JS-side tests mock `requireNativeModule` and verify the wrapper's
-contract: method shape, boundary validation, error propagation,
-idempotency. Native-side tests live under
-`android/src/test/kotlin/...` (TODO when the Kotlin module is wired
-to the real AAR).

package/android/build.gradle

@@ -41,7 +41,7 @@ dependencies {
   // Existing Smile ID native AAR — the entire ML pipeline lives here.
   // This Expo Module is a thin Kotlin shim that constructs the
   // session and bridges its four methods to JS via Expo Modules.
-  implementation 'com.usesmileid:mlkit-face:12.0.0-beta01'
+  implementation 'com.usesmileid:mlkit-face:12.0.0'
 
   // VisionCamera frame-processor plugin classes. The host app provides
   // this dependency via RN autolinking — react-native-vision-camera >= 4.7.3

package/android/src/main/java/com/usesmileid/rn/analyzer/mlkitface/ResizingFrameStorage.kt

@@ -6,6 +6,9 @@
 
 package com.usesmileid.rn.analyzer.mlkitface
 
+import com.usesmileid.bridge.model.BoundingBox as NativeBoundingBox
+import com.usesmileid.bridge.model.CaptureType
+import com.usesmileid.bridge.model.ViewportCropRegion as NativeViewportCropRegion
 import com.usesmileid.bridge.model.YuvFrame
 import com.usesmileid.bridge.storage.DocumentSide
 import com.usesmileid.bridge.storage.FrameKind
@@ -16,6 +19,7 @@ import java.io.File
 import java.io.IOException
 import java.util.concurrent.ConcurrentLinkedDeque
 import java.util.concurrent.atomic.AtomicReference
+import kotlin.math.roundToInt
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
@@ -32,8 +36,15 @@ import kotlinx.coroutines.withContext
  * carries no override, so the on-disk pixel size always matches the
  * `image_width` / `image_height` values reported in job metadata.
  *
- * Document frames are not resized — they are written at the analyzer's native
- * frame resolution using the specified [documentQuality]. Document dimensions
+ * Document frames are decoded upright, then cropped to the guide rectangle when
+ * a crop region has been published via [setCropRegion] (the on-screen frame the
+ * user aligned the document into) so the uploaded image matches the capture
+ * preview. The crop runs at full resolution first; the resulting crop is then
+ * downscaled so its longest edge is bounded at [DOCUMENT_MAX_DIMENSION] (1600
+ * px), preserving aspect ratio and never upscaling, before JPEG-encoding at the
+ * specified [documentQuality]. Cropping before the downscale keeps maximum
+ * detail in the document region — the bound applies to the crop, not the whole
+ * frame, so a tightly framed card fills the saved pixels. Document dimensions
  * are not part of the token `sdk_config` contract.
  *
  * Resize and encode run on a private `Dispatchers.IO` scope; [persist] is
@@ -42,6 +53,7 @@ import kotlinx.coroutines.withContext
  */
 internal class ResizingFrameStorage(
     private val sessionDirectory: File,
+    private val captureType: CaptureType,
     private val documentSide: DocumentSide = DocumentSide.Front,
     private val selfieTargetWidth: Int = 480,
     private val selfieTargetHeight: Int = 640,
@@ -52,10 +64,34 @@ internal class ResizingFrameStorage(
     private val documentQuality: Int = 85,
 ) : FrameStorage {
 
+    companion object {
+        /**
+         * Longest-edge bound, in pixels, for the saved document JPEG. The
+         * guide-rect crop is downscaled so its larger dimension never exceeds
+         * this value, preserving aspect ratio and never upscaling.
+         */
+        private const val DOCUMENT_MAX_DIMENSION = 1600
+
+        /**
+         * Outward padding fraction applied to the guide-rect crop on each side
+         * before cropping. Fifteen percent of the box's own width/height keeps
+         * the document's corners and edges from being clipped when the card is
+         * framed tight against the on-screen guide.
+         */
+        private const val DOCUMENT_CROP_PADDING_FRACTION = 0.15f
+    }
+
     private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
     private val pending = ConcurrentLinkedDeque<Job>()
     private val firstError = AtomicReference<Throwable?>(null)
 
+    @Volatile
+    private var cropRegion: NativeViewportCropRegion? = null
+
+    override fun setCropRegion(region: NativeViewportCropRegion?) {
+        cropRegion = region
+    }
+
     override fun fileFor(kind: FrameKind, captureIndex: Int): File = File(
         sessionDirectory,
         when (kind) {
@@ -69,9 +105,21 @@ internal class ResizingFrameStorage(
     )
 
     override fun persist(frame: YuvFrame, file: File) {
+        launchEncode(frame, file, documentCropRegion = null)
+    }
+
+    override fun persistDocument(frame: YuvFrame, file: File, liveBoundingBox: NativeBoundingBox?) {
+        // Snapshot the crop region at dispatch time so a concurrent
+        // setCropRegion(null) between here and the coroutine body can't tear
+        // the state. liveBoundingBox is unused: the guide rectangle the user
+        // framed (cropRegion) takes precedence over the live detector box.
+        launchEncode(frame, file, documentCropRegion = cropRegion)
+    }
+
+    private fun launchEncode(frame: YuvFrame, file: File, documentCropRegion: NativeViewportCropRegion?) {
         val job = scope.launch {
             try {
-                encodeAndWrite(frame, file)
+                encodeAndWrite(frame, file, documentCropRegion)
             } catch (throwable: Throwable) {
                 firstError.compareAndSet(null, throwable)
                 throw throwable
@@ -89,10 +137,33 @@ internal class ResizingFrameStorage(
     }
 
     override fun clear() {
-        sessionDirectory.listFiles()?.forEach { it.delete() }
+        // Selfie and document captures share one session directory; delete
+        // only this role's own frames.
+        sessionDirectory.listFiles()?.forEach { file ->
+            if (ownsForClear(file)) file.delete()
+        }
     }
 
-    private suspend fun encodeAndWrite(frame: YuvFrame, file: File) {
+    private fun ownsForClear(file: File): Boolean = when (captureType) {
+        CaptureType.SELFIE -> {
+            val kind = frameKindOf(file)
+            kind == FrameKind.Selfie || kind == FrameKind.Liveness
+        }
+        CaptureType.DOCUMENT -> file.name == fileFor(FrameKind.Document, captureIndex = 0).name
+    }
+
+    private fun frameKindOf(file: File): FrameKind? = when {
+        file.name.startsWith("selfie") -> FrameKind.Selfie
+        file.name.startsWith("liveness") -> FrameKind.Liveness
+        file.name.startsWith("document") -> FrameKind.Document
+        else -> null
+    }
+
+    private suspend fun encodeAndWrite(
+        frame: YuvFrame,
+        file: File,
+        documentCropRegion: NativeViewportCropRegion?,
+    ) {
         withContext(Dispatchers.IO) {
             file.parentFile?.mkdirs()
             val name = file.name
@@ -116,16 +187,61 @@ internal class ResizingFrameStorage(
                     }
                 }
                 else -> {
-                    // Document frames: encode at native resolution, quality only.
-                    val bitmap = frame.toArgbBitmap(jpegQuality = documentQuality)
+                    // Document frames: decode to the upright bitmap, crop to the guide
+                    // rectangle (if a crop region is set) at full resolution, then
+                    // downscale the crop so its longest edge is bounded at
+                    // DOCUMENT_MAX_DIMENSION. Cropping before the downscale keeps maximum
+                    // detail in the document region — the bound applies to the crop, not
+                    // the whole frame, so a tightly framed card fills the saved pixels.
+                    // toFrameBoundingBox returns a normalized 0–1 box in the upright
+                    // frame's coordinate space, mapped to pixels by the bitmap dimensions.
+                    // The box is padded outward by DOCUMENT_CROP_PADDING_FRACTION of its
+                    // own size on each side so the document's corners and edges keep
+                    // breathing room and are never clipped when the card is framed tight
+                    // against the guide.
+                    val source = frame.toArgbBitmap(jpegQuality = documentQuality)
                         ?: throw IOException("toArgbBitmap returned null for ${file.name}")
+                    val cropped = if (documentCropRegion != null) {
+                        val box = documentCropRegion.toFrameBoundingBox(source.width, source.height)
+                        val padX = box.width * DOCUMENT_CROP_PADDING_FRACTION
+                        val padY = box.height * DOCUMENT_CROP_PADDING_FRACTION
+                        // Clamp left/top to leave room for at least one pixel and derive
+                        // right/bottom from the clamped left/top so the crop never runs past
+                        // the bitmap edge. Bitmap.createBitmap throws IllegalArgumentException
+                        // when x + width exceeds the source width, which is reachable if the
+                        // padded box lands fully off an edge and both bounds collapse onto the
+                        // same coordinate.
+                        val pixelLeft = ((box.left - padX) * source.width).roundToInt().coerceIn(0, source.width - 1)
+                        val pixelTop = ((box.top - padY) * source.height).roundToInt().coerceIn(0, source.height - 1)
+                        val pixelRight = ((box.left + box.width + padX) * source.width).roundToInt().coerceIn(pixelLeft + 1, source.width)
+                        val pixelBottom = ((box.top + box.height + padY) * source.height).roundToInt().coerceIn(pixelTop + 1, source.height)
+                        val cropWidth = (pixelRight - pixelLeft).coerceAtLeast(1)
+                        val cropHeight = (pixelBottom - pixelTop).coerceAtLeast(1)
+                        android.graphics.Bitmap.createBitmap(source, pixelLeft, pixelTop, cropWidth, cropHeight)
+                    } else {
+                        source
+                    }
+                    // Downscale the crop so its longest edge is at most
+                    // DOCUMENT_MAX_DIMENSION, preserving aspect ratio. Skip when the crop
+                    // already fits so a small document is never upscaled.
+                    val longest = maxOf(cropped.width, cropped.height)
+                    val scaled = if (longest > DOCUMENT_MAX_DIMENSION) {
+                        val scale = DOCUMENT_MAX_DIMENSION.toFloat() / longest
+                        val scaledWidth = (cropped.width * scale).roundToInt().coerceAtLeast(1)
+                        val scaledHeight = (cropped.height * scale).roundToInt().coerceAtLeast(1)
+                        android.graphics.Bitmap.createScaledBitmap(cropped, scaledWidth, scaledHeight, true)
+                    } else {
+                        cropped
+                    }
                     try {
                         val success = file.outputStream().use {
-                            bitmap.compress(android.graphics.Bitmap.CompressFormat.JPEG, documentQuality, it)
+                            scaled.compress(android.graphics.Bitmap.CompressFormat.JPEG, documentQuality, it)
                         }
                         if (!success) throw IOException("Bitmap.compress(JPEG) failed for ${file.absolutePath}")
                     } finally {
-                        bitmap.recycle()
+                        if (scaled !== cropped) scaled.recycle()
+                        if (cropped !== source) cropped.recycle()
+                        source.recycle()
                     }
                 }
             }

package/android/src/main/java/com/usesmileid/rn/analyzer/mlkitface/UseSmileIDAnalyzerExecutor.kt

@@ -8,6 +8,7 @@ package com.usesmileid.rn.analyzer.mlkitface
 
 import android.content.Context
 import com.usesmileid.bridge.model.CaptureReadyEvent as NativeCaptureReadyEvent
+import com.usesmileid.bridge.model.CaptureType
 import com.usesmileid.bridge.model.YuvFrame as NativeYuvFrame
 import com.usesmileid.bridge.storage.DocumentSide
 import com.usesmileid.bridge.storage.FrameStorage
@@ -74,7 +75,7 @@ abstract class UseSmileIDAnalyzerExecutor<
     protected abstract fun createSession(config: PConfig, frameStorage: FrameStorage): Session
 
     /**
-     * The document side this session's [JpegFrameStorage] writes, selecting
+     * The document side this session's [ResizingFrameStorage] writes, selecting
      * the `document_front.jpg` vs `document_back.jpg` filename. Face modules
      * keep the default ([DocumentSide.Front]); document modules override to
      * read the side from their wire config so two-sided captures don't
@@ -260,6 +261,7 @@ abstract class UseSmileIDAnalyzerExecutor<
         config: Map<*, *>?,
     ): ResizingFrameStorage = ResizingFrameStorage(
         sessionDirectory = sessionDir,
+        captureType = CaptureType.SELFIE,
         documentSide = documentSide,
         selfieTargetWidth = (config?.get("selfieImageWidth") as? Number)?.toInt() ?: 480,
         selfieTargetHeight = (config?.get("selfieImageHeight") as? Number)?.toInt() ?: 640,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@smileid/usesmileid_mlkit_face",
-  "version": "12.0.0-beta01",
-  "description": "UseSmileID v12 — Google ML Kit face provider (Android-only). Thin Expo Module shim around the existing native AAR `com.smileidentity:mlkit-face`.",
+  "version": "12.0.0",
+  "description": "Android MLKit face detection implementation for the UseSmileID React Native (Expo) SDK.",
   "homepage": "https://usesmileid.com",
   "repository": {
     "type": "git",
@@ -43,8 +43,8 @@
     "CHANGELOG.md"
   ],
   "dependencies": {
-    "@smileid/usesmileid_bridge": "12.0.0-beta01",
-    "@smileid/usesmileid_platform_interface": "12.0.0-beta01"
+    "@smileid/usesmileid_bridge": "12.0.0",
+    "@smileid/usesmileid_platform_interface": "12.0.0"
   },
   "peerDependencies": {
     "expo": ">=56.0.0",
@@ -57,8 +57,7 @@
     "typescript": "^6.0.3"
   },
   "publishConfig": {
-    "access": "public",
-    "provenance": false
+    "access": "public"
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",