capacitor-card-scanner 0.0.1 → 0.2.0

package/README.md

@@ -45,13 +45,16 @@ Add to the app's `Info.plist`:
 1. Add a CSS class to the Theme / screen:
 
    ```css
+   html:has(body.camera-active),
    body.camera-active,
+   body.camera-active #reactContainer,
+   body.camera-active .screen-container,
+   body.camera-active .active-screen,
+   body.camera-active .screen,
    body.camera-active .layout,
    body.camera-active .main,
    body.camera-active .main-content,
-   body.camera-active .content,
-   body.camera-active .screen,
-   body.camera-active .active-screen {
+   body.camera-active .content {
      background: transparent !important;
    }
    ```
@@ -78,6 +81,25 @@ Add to the app's `Info.plist`:
    $parameters.ImageBase64 = photo.base64;
    ```
 
+### Card detection (Android)
+
+After `start()`, call `startDetection()`. A red rectangle is drawn around the
+card (between the camera and the WebView) and its text is read on-device with
+ML Kit. `cardDetected` fires once per card, when the same collector number (or
+name) was read on `stableFrames` consecutive runs. Remove the card from view
+for ~1.5s to scan the same card again.
+
+```js
+const { CardScanner } = Capacitor.Plugins;
+await CardScanner.addListener('cardDetected', (card) => {
+  // card.name, card.collectorNumber ('025/165'), card.text, card.lines, card.cardBox, card.image
+});
+await CardScanner.startDetection({ intervalMs: 300, stableFrames: 2, includeImage: false });
+// ...
+await CardScanner.stopDetection();
+await CardScanner.removeAllListeners();
+```
+
 ## API
 
 <docgen-index>
@@ -88,6 +110,10 @@ Add to the app's `Info.plist`:
 * [`setTorch(...)`](#settorch)
 * [`setZoom(...)`](#setzoom)
 * [`flip()`](#flip)
+* [`startDetection(...)`](#startdetection)
+* [`stopDetection()`](#stopdetection)
+* [`addListener('cardDetected', ...)`](#addlistenercarddetected-)
+* [`removeAllListeners()`](#removealllisteners)
 * [`checkPermissions()`](#checkpermissions)
 * [`requestPermissions()`](#requestpermissions)
 * [Interfaces](#interfaces)
@@ -184,6 +210,60 @@ Switches between the rear and front camera.
 --------------------
 
 
+### startDetection(...)
+
+```typescript
+startDetection(options?: DetectionOptions | undefined) => Promise<void>
+```
+
+Starts looking for cards: draws a red rectangle around the card and reads
+its text, emitting `cardDetected`. Requires `start()` first. Android only.
+
+| Param         | Type                                                          |
+| ------------- | ------------------------------------------------------------- |
+| **`options`** | <code><a href="#detectionoptions">DetectionOptions</a></code> |
+
+--------------------
+
+
+### stopDetection()
+
+```typescript
+stopDetection() => Promise<void>
+```
+
+Stops card detection and hides the rectangle.
+
+--------------------
+
+
+### addListener('cardDetected', ...)
+
+```typescript
+addListener(eventName: 'cardDetected', listener: (event: CardDetectedEvent) => void) => Promise<PluginListenerHandle>
+```
+
+Fired once per card, when its text has been read consistently.
+
+| Param           | Type                                                                                |
+| --------------- | ----------------------------------------------------------------------------------- |
+| **`eventName`** | <code>'cardDetected'</code>                                                         |
+| **`listener`**  | <code>(event: <a href="#carddetectedevent">CardDetectedEvent</a>) =&gt; void</code> |
+
+**Returns:** <code>Promise&lt;<a href="#pluginlistenerhandle">PluginListenerHandle</a>&gt;</code>
+
+--------------------
+
+
+### removeAllListeners()
+
+```typescript
+removeAllListeners() => Promise<void>
+```
+
+--------------------
+
+
 ### checkPermissions()
 
 ```typescript
@@ -246,6 +326,59 @@ requestPermissions() => Promise<PermissionStatus>
 | **`ratio`** | <code>number</code> | Zoom ratio, clamped between the camera's min and max. |
 
 
+#### DetectionOptions
+
+| Prop                  | Type                 | Description                                                                                                                     | Default                         |
+| --------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- |
+| **`intervalMs`**      | <code>number</code>  | Minimum time between two OCR runs, in milliseconds. The red rectangle is updated on every frame regardless.                     | <code>300</code>                |
+| **`stableFrames`**    | <code>number</code>  | How many consecutive OCR runs must read the same card before `cardDetected` fires.                                              | <code>2</code>                  |
+| **`includeImage`**    | <code>boolean</code> | Include a JPEG of the detected card in the event.                                                                               | <code>false</code>              |
+| **`cardAspectRatio`** | <code>number</code>  | Expected width / height of an upright card. Only upright cards are detected; a card lying sideways falls outside the tolerance. | <code>0.716 (63 x 88 mm)</code> |
+| **`aspectTolerance`** | <code>number</code>  | Allowed relative deviation from `cardAspectRatio` (0.15 = ±15%). Increase it if tilted cards are not detected.                  | <code>0.15</code>               |
+| **`minCardArea`**     | <code>number</code>  | Smallest card size, as a fraction of the camera frame (0-1).                                                                    | <code>0.08</code>               |
+| **`maxCardArea`**     | <code>number</code>  | Largest card size, as a fraction of the camera frame (0-1).                                                                     | <code>0.9</code>                |
+| **`requireCard`**     | <code>boolean</code> | Only read text when a card-shaped object is visible. When false, the whole frame is read if no card is found.                   | <code>true</code>               |
+
+
+#### PluginListenerHandle
+
+| Prop         | Type                                      |
+| ------------ | ----------------------------------------- |
+| **`remove`** | <code>() =&gt; Promise&lt;void&gt;</code> |
+
+
+#### CardDetectedEvent
+
+| Prop                  | Type                                | Description                                                                  |
+| --------------------- | ----------------------------------- | ---------------------------------------------------------------------------- |
+| **`name`**            | <code>string</code>                 | Best guess for the card name (topmost line of text on the card).             |
+| **`collectorNumber`** | <code>string</code>                 | Collector number, e.g. `025/165`.                                            |
+| **`text`**            | <code>string</code>                 | All text read on the card.                                                   |
+| **`lines`**           | <code>TextLine[]</code>             |                                                                              |
+| **`cardBox`**         | <code><a href="#box">Box</a></code> | Where the card is in the camera frame. Missing if no card outline was found. |
+| **`image`**           | <code>string</code>                 | Base64 JPEG of the card, when `includeImage` is true.                        |
+
+
+#### TextLine
+
+| Prop       | Type                                | Description                     |
+| ---------- | ----------------------------------- | ------------------------------- |
+| **`text`** | <code>string</code>                 |                                 |
+| **`box`**  | <code><a href="#box">Box</a></code> | Position inside the card image. |
+
+
+#### Box
+
+Normalized rectangle (0-1).
+
+| Prop         | Type                |
+| ------------ | ------------------- |
+| **`x`**      | <code>number</code> |
+| **`y`**      | <code>number</code> |
+| **`width`**  | <code>number</code> |
+| **`height`** | <code>number</code> |
+
+
 #### PermissionStatus
 
 | Prop         | Type                                                        |

package/android/build.gradle

@@ -61,4 +61,6 @@ dependencies {
     implementation "androidx.camera:camera-camera2:$camerax_version"
     implementation "androidx.camera:camera-lifecycle:$camerax_version"
     implementation "androidx.camera:camera-view:$camerax_version"
+    implementation "com.google.mlkit:text-recognition:16.0.1"
+    implementation "com.google.mlkit:object-detection:17.0.2"
 }

package/android/src/main/java/pt/tcgclassifier/cardscanner/CardDetector.java

@@ -0,0 +1,283 @@
+package pt.tcgclassifier.cardscanner;
+
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.graphics.Rect;
+import android.graphics.RectF;
+import android.os.SystemClock;
+import android.util.Base64;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import androidx.camera.core.ImageAnalysis;
+import androidx.camera.core.ImageProxy;
+import com.getcapacitor.JSArray;
+import com.getcapacitor.JSObject;
+import com.getcapacitor.Logger;
+import com.google.android.gms.tasks.Tasks;
+import com.google.mlkit.vision.common.InputImage;
+import com.google.mlkit.vision.objects.DetectedObject;
+import com.google.mlkit.vision.objects.ObjectDetection;
+import com.google.mlkit.vision.objects.ObjectDetector;
+import com.google.mlkit.vision.objects.defaults.ObjectDetectorOptions;
+import com.google.mlkit.vision.text.Text;
+import com.google.mlkit.vision.text.TextRecognition;
+import com.google.mlkit.vision.text.TextRecognizer;
+import com.google.mlkit.vision.text.latin.TextRecognizerOptions;
+import java.io.ByteArrayOutputStream;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Locates the card in every camera frame (for the on-screen rectangle) and,
+ * every {@code intervalMs}, runs OCR inside it. A card is reported once the
+ * same collector number (or name) has been read on several consecutive runs.
+ */
+public class CardDetector implements ImageAnalysis.Analyzer {
+
+    public interface Listener {
+        void onCardDetected(JSObject result);
+    }
+
+    /** Tuning knobs, all overridable from JS through startDetection(). */
+    public static class Options {
+
+        /** Minimum time between two OCR runs. */
+        public long intervalMs = 300;
+        /** Consecutive OCR runs that must agree before a card is reported. */
+        public int stableFrames = 2;
+        /** Include a JPEG of the card in the event. */
+        public boolean includeImage = false;
+        /** Expected width / height of an upright card (63 x 88 mm). */
+        public float cardAspectRatio = 63f / 88f;
+        /** Allowed relative deviation from cardAspectRatio (0.15 = +/-15%). */
+        public float aspectTolerance = 0.15f;
+        /** Smallest card size, as a fraction of the frame area. */
+        public float minCardArea = 0.08f;
+        /** Largest card size, as a fraction of the frame area. */
+        public float maxCardArea = 0.9f;
+        /** Only run OCR when a card-shaped object is visible. */
+        public boolean requireCard = true;
+    }
+
+    public interface BoxListener {
+        /**
+         * @param box card bounds normalized (0-1) to the upright frame, or null when no card is visible
+         */
+        void onBox(@Nullable RectF box, int frameWidth, int frameHeight);
+    }
+
+    // e.g. "025/165", "SV001/SV122", "TG05/TG30", "GG01/GG70"
+    private static final Pattern NUMBER = Pattern.compile("\\b([A-Za-z]{0,4}\\d{1,3}[a-z]?)\\s*/\\s*([A-Za-z]{0,4}\\d{1,3})\\b");
+    private static final Pattern HEADER = Pattern.compile(
+        "^(basic|stage\\s*\\d|trainer|item|supporter|stadium|energy|pok[eé]mon|evolves|hp\\b)",
+        Pattern.CASE_INSENSITIVE
+    );
+    private static final Pattern HP_SUFFIX = Pattern.compile("\\s*HP\\s*\\d+\\s*$", Pattern.CASE_INSENSITIVE);
+    private static final Pattern LETTER = Pattern.compile("\\p{L}");
+    /** After this long without reading a card, the same card can be reported again. */
+    private static final long RESET_AFTER_MS = 1500;
+    /** Extra margin around the detected card before running OCR. */
+    private static final float CROP_MARGIN = 0.04f;
+
+    private final ObjectDetector objectDetector = ObjectDetection.getClient(
+        new ObjectDetectorOptions.Builder().setDetectorMode(ObjectDetectorOptions.STREAM_MODE).build()
+    );
+    private final TextRecognizer recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS);
+    private final Options options;
+    private final Listener listener;
+    private volatile BoxListener boxListener;
+
+    // Only touched from the analyzer thread.
+    private long lastOcrAt;
+    private long lastSeenAt;
+    private String candidateKey;
+    private int candidateCount;
+    private String lastEmittedKey;
+
+    public CardDetector(Options options, Listener listener) {
+        this.options = options;
+        this.options.stableFrames = Math.max(1, options.stableFrames);
+        this.listener = listener;
+    }
+
+    public void setBoxListener(BoxListener boxListener) {
+        this.boxListener = boxListener;
+    }
+
+    public void close() {
+        objectDetector.close();
+        recognizer.close();
+    }
+
+    @Override
+    public void analyze(@NonNull ImageProxy image) {
+        long now = SystemClock.elapsedRealtime();
+        int rotation = image.getImageInfo().getRotationDegrees();
+        Bitmap raw;
+        try {
+            raw = image.toBitmap();
+        } finally {
+            image.close();
+        }
+        int frameW = rotation % 180 == 0 ? raw.getWidth() : raw.getHeight();
+        int frameH = rotation % 180 == 0 ? raw.getHeight() : raw.getWidth();
+
+        try {
+            List<DetectedObject> objects = Tasks.await(objectDetector.process(InputImage.fromBitmap(raw, rotation)));
+            Rect card = pickCard(objects, frameW, frameH);
+            RectF cardBox = card == null ? null : new RectF(
+                card.left / (float) frameW,
+                card.top / (float) frameH,
+                card.right / (float) frameW,
+                card.bottom / (float) frameH
+            );
+            BoxListener bl = boxListener;
+            if (bl != null) bl.onBox(cardBox, frameW, frameH);
+
+            if (now - lastOcrAt < options.intervalMs) return;
+            lastOcrAt = now;
+
+            if (card == null && options.requireCard) {
+                noCardRead(now);
+                return;
+            }
+
+            Bitmap upright = rotate(raw, rotation);
+            Bitmap region = card == null ? upright : crop(upright, card);
+            Text text = Tasks.await(recognizer.process(InputImage.fromBitmap(region, 0)));
+            handle(text, region, cardBox, now);
+        } catch (Exception e) {
+            Logger.error("CardDetector", "Detection failed", e);
+        }
+    }
+
+    /** The biggest detected object with the size and upright proportions of a card. */
+    @Nullable
+    private Rect pickCard(List<DetectedObject> objects, int frameW, int frameH) {
+        float frameArea = (float) frameW * frameH;
+        float minRatio = options.cardAspectRatio * (1f - options.aspectTolerance);
+        float maxRatio = options.cardAspectRatio * (1f + options.aspectTolerance);
+        Rect best = null;
+        long bestArea = 0;
+        for (DetectedObject o : objects) {
+            Rect r = new Rect(o.getBoundingBox());
+            if (!r.intersect(0, 0, frameW, frameH) || r.height() == 0) continue;
+
+            long area = (long) r.width() * r.height();
+            float fraction = area / frameArea;
+            if (fraction < options.minCardArea || fraction > options.maxCardArea) continue;
+
+            float ratio = r.width() / (float) r.height();
+            if (ratio < minRatio || ratio > maxRatio) continue;
+
+            if (area > bestArea) {
+                best = r;
+                bestArea = area;
+            }
+        }
+        return best;
+    }
+
+    private static Bitmap rotate(Bitmap bitmap, int rotation) {
+        if (rotation == 0) return bitmap;
+        Matrix matrix = new Matrix();
+        matrix.postRotate(rotation);
+        return Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
+    }
+
+    private static Bitmap crop(Bitmap bitmap, Rect card) {
+        int mx = Math.round(card.width() * CROP_MARGIN);
+        int my = Math.round(card.height() * CROP_MARGIN);
+        int left = Math.max(0, card.left - mx);
+        int top = Math.max(0, card.top - my);
+        int right = Math.min(bitmap.getWidth(), card.right + mx);
+        int bottom = Math.min(bitmap.getHeight(), card.bottom + my);
+        return Bitmap.createBitmap(bitmap, left, top, right - left, bottom - top);
+    }
+
+    private void handle(Text text, Bitmap region, @Nullable RectF cardBox, long now) {
+        float w = region.getWidth();
+        float h = region.getHeight();
+        JSArray lines = new JSArray();
+        String number = null;
+        String name = null;
+        int nameTop = Integer.MAX_VALUE;
+
+        for (Text.TextBlock block : text.getTextBlocks()) {
+            for (Text.Line line : block.getLines()) {
+                String s = line.getText().trim();
+                Rect r = line.getBoundingBox();
+                if (s.isEmpty() || r == null) continue;
+
+                JSObject item = new JSObject();
+                item.put("text", s);
+                item.put("box", box(r.left / w, r.top / h, r.width() / w, r.height() / h));
+                lines.put(item);
+
+                if (number == null) {
+                    Matcher m = NUMBER.matcher(s);
+                    if (m.find()) number = m.group(1) + "/" + m.group(2);
+                }
+                if (r.top < nameTop && isNameCandidate(s)) {
+                    name = HP_SUFFIX.matcher(s).replaceAll("").trim();
+                    nameTop = r.top;
+                }
+            }
+        }
+
+        String key = number != null ? number : name;
+        if (key == null) {
+            noCardRead(now);
+            return;
+        }
+        lastSeenAt = now;
+
+        if (key.equals(candidateKey)) {
+            candidateCount++;
+        } else {
+            candidateKey = key;
+            candidateCount = 1;
+        }
+        if (candidateCount < options.stableFrames || key.equals(lastEmittedKey)) return;
+        lastEmittedKey = key;
+
+        JSObject result = new JSObject();
+        result.put("name", name);
+        result.put("collectorNumber", number);
+        result.put("text", text.getText());
+        result.put("lines", lines);
+        if (cardBox != null) {
+            result.put("cardBox", box(cardBox.left, cardBox.top, cardBox.width(), cardBox.height()));
+        }
+        if (options.includeImage) {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            region.compress(Bitmap.CompressFormat.JPEG, 80, out);
+            result.put("image", Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP));
+        }
+        listener.onCardDetected(result);
+    }
+
+    private void noCardRead(long now) {
+        candidateKey = null;
+        candidateCount = 0;
+        if (now - lastSeenAt > RESET_AFTER_MS) lastEmittedKey = null;
+    }
+
+    private static JSObject box(float x, float y, float width, float height) {
+        JSObject box = new JSObject();
+        box.put("x", x);
+        box.put("y", y);
+        box.put("width", width);
+        box.put("height", height);
+        return box;
+    }
+
+    private static boolean isNameCandidate(String s) {
+        if (HEADER.matcher(s).find()) return false;
+        Matcher m = LETTER.matcher(s);
+        int letters = 0;
+        while (m.find()) letters++;
+        return letters >= 3;
+    }
+}