capacitor-card-scanner 0.0.1 → 0.2.0

package/android/src/main/java/pt/tcgclassifier/cardscanner/CardScanner.java

@@ -1,195 +1,267 @@
-package pt.tcgclassifier.cardscanner;
-
-import android.graphics.Bitmap;
-import android.graphics.BitmapFactory;
-import android.graphics.Color;
-import android.graphics.Matrix;
-import android.util.Base64;
-import android.view.ViewGroup;
-import android.webkit.WebView;
-import androidx.annotation.NonNull;
-import androidx.appcompat.app.AppCompatActivity;
-import androidx.camera.core.Camera;
-import androidx.camera.core.CameraSelector;
-import androidx.camera.core.ImageCapture;
-import androidx.camera.core.ImageCaptureException;
-import androidx.camera.core.ImageProxy;
-import androidx.camera.core.Preview;
-import androidx.camera.core.ZoomState;
-import androidx.camera.lifecycle.ProcessCameraProvider;
-import androidx.camera.view.PreviewView;
-import androidx.core.content.ContextCompat;
-import com.google.common.util.concurrent.ListenableFuture;
-import java.io.ByteArrayOutputStream;
-import java.nio.ByteBuffer;
-
-/**
- * Places a CameraX preview behind the Capacitor WebView and makes the WebView
- * transparent, so the HTML UI is drawn on top of the camera feed.
- */
-public class CardScanner {
-
-    public interface Callback<T> {
-        void onSuccess(T result);
-        void onError(String message, Exception e);
-    }
-
-    public static class Photo {
-
-        public final String base64;
-        public final int width;
-        public final int height;
-
-        Photo(String base64, int width, int height) {
-            this.base64 = base64;
-            this.width = width;
-            this.height = height;
-        }
-    }
-
-    private final AppCompatActivity activity;
-    private final WebView webView;
-
-    private PreviewView previewView;
-    private ProcessCameraProvider cameraProvider;
-    private ImageCapture imageCapture;
-    private Camera camera;
-    private boolean useFront = false;
-
-    public CardScanner(AppCompatActivity activity, WebView webView) {
-        this.activity = activity;
-        this.webView = webView;
-    }
-
-    public boolean isRunning() {
-        return camera != null;
-    }
-
-    public boolean isFront() {
-        return useFront;
-    }
-
-    /** Must be called on the UI thread. */
-    public void start(boolean front, Callback<Void> callback) {
-        useFront = front;
-
-        if (previewView == null) {
-            previewView = new PreviewView(activity);
-            // TextureView-based mode composes correctly underneath a transparent WebView.
-            previewView.setImplementationMode(PreviewView.ImplementationMode.COMPATIBLE);
-            previewView.setScaleType(PreviewView.ScaleType.FILL_CENTER);
-            ViewGroup parent = (ViewGroup) webView.getParent();
-            parent.addView(previewView, 0, new ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT));
-        }
-
-        webView.setBackgroundColor(Color.TRANSPARENT);
-
-        ListenableFuture<ProcessCameraProvider> future = ProcessCameraProvider.getInstance(activity);
-        future.addListener(
-            () -> {
-                try {
-                    cameraProvider = future.get();
-                    bind();
-                    callback.onSuccess(null);
-                } catch (Exception e) {
-                    callback.onError("Failed to start camera: " + e.getMessage(), e);
-                }
-            },
-            ContextCompat.getMainExecutor(activity)
-        );
-    }
-
-    private void bind() {
-        cameraProvider.unbindAll();
-
-        Preview preview = new Preview.Builder().build();
-        preview.setSurfaceProvider(previewView.getSurfaceProvider());
-
-        imageCapture = new ImageCapture.Builder().setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY).build();
-
-        CameraSelector selector = useFront ? CameraSelector.DEFAULT_FRONT_CAMERA : CameraSelector.DEFAULT_BACK_CAMERA;
-        camera = cameraProvider.bindToLifecycle(activity, selector, preview, imageCapture);
-    }
-
-    /** Must be called on the UI thread. */
-    public void stop() {
-        if (cameraProvider != null) {
-            cameraProvider.unbindAll();
-        }
-        camera = null;
-        imageCapture = null;
-        if (previewView != null) {
-            ((ViewGroup) previewView.getParent()).removeView(previewView);
-            previewView = null;
-        }
-        webView.setBackgroundColor(Color.WHITE);
-    }
-
-    /** Must be called on the UI thread. */
-    public void flip(Callback<Void> callback) {
-        useFront = !useFront;
-        try {
-            bind();
-            callback.onSuccess(null);
-        } catch (Exception e) {
-            callback.onError("Failed to flip camera: " + e.getMessage(), e);
-        }
-    }
-
-    public void setTorch(boolean enabled) {
-        camera.getCameraControl().enableTorch(enabled);
-    }
-
-    public void setZoom(float ratio) {
-        ZoomState state = camera.getCameraInfo().getZoomState().getValue();
-        if (state != null) {
-            ratio = Math.max(state.getMinZoomRatio(), Math.min(state.getMaxZoomRatio(), ratio));
-        }
-        camera.getCameraControl().setZoomRatio(ratio);
-    }
-
-    public void capture(int quality, Callback<Photo> callback) {
-        imageCapture.takePicture(
-            ContextCompat.getMainExecutor(activity),
-            new ImageCapture.OnImageCapturedCallback() {
-                @Override
-                public void onCaptureSuccess(@NonNull ImageProxy image) {
-                    try {
-                        callback.onSuccess(toPhoto(image, quality));
-                    } catch (Exception e) {
-                        callback.onError("Failed to process image: " + e.getMessage(), e);
-                    } finally {
-                        image.close();
-                    }
-                }
-
-                @Override
-                public void onError(@NonNull ImageCaptureException e) {
-                    callback.onError("Capture failed: " + e.getMessage(), e);
-                }
-            }
-        );
-    }
-
-    private Photo toPhoto(ImageProxy image, int quality) {
-        ByteBuffer buffer = image.getPlanes()[0].getBuffer();
-        byte[] jpeg = new byte[buffer.remaining()];
-        buffer.get(jpeg);
-
-        Bitmap bitmap = BitmapFactory.decodeByteArray(jpeg, 0, jpeg.length);
-        int rotation = image.getImageInfo().getRotationDegrees();
-        if (rotation != 0) {
-            Matrix matrix = new Matrix();
-            matrix.postRotate(rotation);
-            Bitmap rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
-            bitmap.recycle();
-            bitmap = rotated;
-        }
-
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        bitmap.compress(Bitmap.CompressFormat.JPEG, quality, out);
-        Photo photo = new Photo(Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP), bitmap.getWidth(), bitmap.getHeight());
-        bitmap.recycle();
-        return photo;
-    }
-}
+package pt.tcgclassifier.cardscanner;
+
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.Color;
+import android.graphics.Matrix;
+import android.util.Base64;
+import android.util.Size;
+import android.view.ViewGroup;
+import android.webkit.WebView;
+import androidx.annotation.NonNull;
+import androidx.appcompat.app.AppCompatActivity;
+import androidx.camera.core.Camera;
+import androidx.camera.core.CameraSelector;
+import androidx.camera.core.ImageAnalysis;
+import androidx.camera.core.ImageCapture;
+import androidx.camera.core.ImageCaptureException;
+import androidx.camera.core.ImageProxy;
+import androidx.camera.core.Preview;
+import androidx.camera.core.UseCase;
+import androidx.camera.core.ZoomState;
+import androidx.camera.core.resolutionselector.ResolutionSelector;
+import androidx.camera.core.resolutionselector.ResolutionStrategy;
+import androidx.camera.lifecycle.ProcessCameraProvider;
+import androidx.camera.view.PreviewView;
+import androidx.core.content.ContextCompat;
+import com.google.common.util.concurrent.ListenableFuture;
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Places a CameraX preview behind the Capacitor WebView and makes the WebView
+ * transparent, so the HTML UI is drawn on top of the camera feed.
+ */
+public class CardScanner {
+
+    public interface Callback<T> {
+        void onSuccess(T result);
+        void onError(String message, Exception e);
+    }
+
+    public static class Photo {
+
+        public final String base64;
+        public final int width;
+        public final int height;
+
+        Photo(String base64, int width, int height) {
+            this.base64 = base64;
+            this.width = width;
+            this.height = height;
+        }
+    }
+
+    private final AppCompatActivity activity;
+    private final WebView webView;
+
+    private PreviewView previewView;
+    private DetectionOverlay overlay;
+    private ProcessCameraProvider cameraProvider;
+    private ImageCapture imageCapture;
+    private Camera camera;
+    private boolean useFront = false;
+
+    private final ExecutorService analysisExecutor = Executors.newSingleThreadExecutor();
+    private CardDetector detector;
+
+    public CardScanner(AppCompatActivity activity, WebView webView) {
+        this.activity = activity;
+        this.webView = webView;
+    }
+
+    public boolean isRunning() {
+        return camera != null;
+    }
+
+    public boolean isFront() {
+        return useFront;
+    }
+
+    /** Must be called on the UI thread. */
+    public void start(boolean front, Callback<Void> callback) {
+        useFront = front;
+
+        if (previewView == null) {
+            previewView = new PreviewView(activity);
+            // TextureView-based mode composes correctly underneath a transparent WebView.
+            previewView.setImplementationMode(PreviewView.ImplementationMode.COMPATIBLE);
+            previewView.setScaleType(PreviewView.ScaleType.FILL_CENTER);
+            ViewGroup parent = (ViewGroup) webView.getParent();
+            parent.addView(previewView, 0, new ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT));
+
+            // Between the preview and the WebView, so HTML controls stay on top of it.
+            overlay = new DetectionOverlay(activity);
+            parent.addView(overlay, 1, new ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT));
+        }
+        overlay.setMirror(front);
+
+        webView.setBackgroundColor(Color.TRANSPARENT);
+
+        ListenableFuture<ProcessCameraProvider> future = ProcessCameraProvider.getInstance(activity);
+        future.addListener(
+            () -> {
+                try {
+                    cameraProvider = future.get();
+                    bind();
+                    callback.onSuccess(null);
+                } catch (Exception e) {
+                    callback.onError("Failed to start camera: " + e.getMessage(), e);
+                }
+            },
+            ContextCompat.getMainExecutor(activity)
+        );
+    }
+
+    private void bind() {
+        cameraProvider.unbindAll();
+
+        Preview preview = new Preview.Builder().build();
+        preview.setSurfaceProvider(previewView.getSurfaceProvider());
+
+        imageCapture = new ImageCapture.Builder().setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY).build();
+
+        List<UseCase> useCases = new ArrayList<>();
+        useCases.add(preview);
+        useCases.add(imageCapture);
+        if (detector != null) {
+            // Collector numbers are tiny, so OCR needs a fairly high resolution.
+            ResolutionSelector resolution = new ResolutionSelector.Builder()
+                .setResolutionStrategy(new ResolutionStrategy(new Size(1920, 1440), ResolutionStrategy.FALLBACK_RULE_CLOSEST_LOWER_THEN_HIGHER))
+                .build();
+            ImageAnalysis analysis = new ImageAnalysis.Builder()
+                .setResolutionSelector(resolution)
+                .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+                .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
+                .build();
+            analysis.setAnalyzer(analysisExecutor, detector);
+            useCases.add(analysis);
+        }
+
+        CameraSelector selector = useFront ? CameraSelector.DEFAULT_FRONT_CAMERA : CameraSelector.DEFAULT_BACK_CAMERA;
+        camera = cameraProvider.bindToLifecycle(activity, selector, useCases.toArray(new UseCase[0]));
+    }
+
+    /** Must be called on the UI thread, with the camera running. */
+    public void startDetection(CardDetector newDetector) {
+        closeDetector();
+        detector = newDetector;
+        DetectionOverlay target = overlay;
+        detector.setBoxListener((box, frameW, frameH) ->
+            target.post(() -> {
+                // Ignore frames that arrive after detection was stopped or restarted.
+                if (detector == newDetector) target.setBox(box, frameW, frameH);
+            })
+        );
+        bind();
+    }
+
+    /** Must be called on the UI thread. */
+    public void stopDetection() {
+        if (detector == null) return;
+        closeDetector();
+        if (isRunning()) bind();
+    }
+
+    private void closeDetector() {
+        if (detector != null) {
+            CardDetector old = detector;
+            detector = null;
+            old.setBoxListener(null);
+            if (overlay != null) overlay.clear();
+            // Let an in-flight analysis finish before releasing the recognizer.
+            analysisExecutor.execute(old::close);
+        }
+    }
+
+    /** Must be called on the UI thread. */
+    public void stop() {
+        closeDetector();
+        if (cameraProvider != null) {
+            cameraProvider.unbindAll();
+        }
+        camera = null;
+        imageCapture = null;
+        if (previewView != null) {
+            ((ViewGroup) previewView.getParent()).removeView(previewView);
+            previewView = null;
+            ((ViewGroup) overlay.getParent()).removeView(overlay);
+            overlay = null;
+        }
+        webView.setBackgroundColor(Color.WHITE);
+    }
+
+    /** Must be called on the UI thread. */
+    public void flip(Callback<Void> callback) {
+        useFront = !useFront;
+        overlay.setMirror(useFront);
+        overlay.clear();
+        try {
+            bind();
+            callback.onSuccess(null);
+        } catch (Exception e) {
+            callback.onError("Failed to flip camera: " + e.getMessage(), e);
+        }
+    }
+
+    public void setTorch(boolean enabled) {
+        camera.getCameraControl().enableTorch(enabled);
+    }
+
+    public void setZoom(float ratio) {
+        ZoomState state = camera.getCameraInfo().getZoomState().getValue();
+        if (state != null) {
+            ratio = Math.max(state.getMinZoomRatio(), Math.min(state.getMaxZoomRatio(), ratio));
+        }
+        camera.getCameraControl().setZoomRatio(ratio);
+    }
+
+    public void capture(int quality, Callback<Photo> callback) {
+        imageCapture.takePicture(
+            ContextCompat.getMainExecutor(activity),
+            new ImageCapture.OnImageCapturedCallback() {
+                @Override
+                public void onCaptureSuccess(@NonNull ImageProxy image) {
+                    try {
+                        callback.onSuccess(toPhoto(image, quality));
+                    } catch (Exception e) {
+                        callback.onError("Failed to process image: " + e.getMessage(), e);
+                    } finally {
+                        image.close();
+                    }
+                }
+
+                @Override
+                public void onError(@NonNull ImageCaptureException e) {
+                    callback.onError("Capture failed: " + e.getMessage(), e);
+                }
+            }
+        );
+    }
+
+    private Photo toPhoto(ImageProxy image, int quality) {
+        ByteBuffer buffer = image.getPlanes()[0].getBuffer();
+        byte[] jpeg = new byte[buffer.remaining()];
+        buffer.get(jpeg);
+
+        Bitmap bitmap = BitmapFactory.decodeByteArray(jpeg, 0, jpeg.length);
+        int rotation = image.getImageInfo().getRotationDegrees();
+        if (rotation != 0) {
+            Matrix matrix = new Matrix();
+            matrix.postRotate(rotation);
+            Bitmap rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
+            bitmap.recycle();
+            bitmap = rotated;
+        }
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        bitmap.compress(Bitmap.CompressFormat.JPEG, quality, out);
+        Photo photo = new Photo(Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP), bitmap.getWidth(), bitmap.getHeight());
+        bitmap.recycle();
+        return photo;
+    }
+}

package/android/src/main/java/pt/tcgclassifier/cardscanner/CardScannerPlugin.java

@@ -88,6 +88,39 @@ public class CardScannerPlugin extends Plugin {
         );
     }
 
+    @PluginMethod
+    public void startDetection(PluginCall call) {
+        if (!ensureRunning(call)) return;
+        CardDetector.Options options = new CardDetector.Options();
+        options.intervalMs = call.getInt("intervalMs", (int) options.intervalMs);
+        options.stableFrames = call.getInt("stableFrames", options.stableFrames);
+        options.includeImage = call.getBoolean("includeImage", options.includeImage);
+        options.cardAspectRatio = call.getFloat("cardAspectRatio", options.cardAspectRatio);
+        options.aspectTolerance = call.getFloat("aspectTolerance", options.aspectTolerance);
+        options.minCardArea = call.getFloat("minCardArea", options.minCardArea);
+        options.maxCardArea = call.getFloat("maxCardArea", options.maxCardArea);
+        options.requireCard = call.getBoolean("requireCard", options.requireCard);
+
+        CardDetector detector = new CardDetector(options, result -> notifyListeners("cardDetected", result));
+        getActivity().runOnUiThread(() -> {
+            try {
+                implementation.startDetection(detector);
+                call.resolve();
+            } catch (Exception e) {
+                detector.close();
+                call.reject("Failed to start detection: " + e.getMessage(), e);
+            }
+        });
+    }
+
+    @PluginMethod
+    public void stopDetection(PluginCall call) {
+        getActivity().runOnUiThread(() -> {
+            implementation.stopDetection();
+            call.resolve();
+        });
+    }
+
     @PluginMethod
     public void setTorch(PluginCall call) {
         if (!ensureRunning(call)) return;

package/android/src/main/java/pt/tcgclassifier/cardscanner/DetectionOverlay.java

@@ -0,0 +1,82 @@
+package pt.tcgclassifier.cardscanner;
+
+import android.content.Context;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+import android.graphics.RectF;
+import android.os.SystemClock;
+import android.view.View;
+import androidx.annotation.Nullable;
+
+/**
+ * Draws a red rectangle around the detected card. Sits between the camera
+ * preview and the (transparent) WebView, and maps frame coordinates the same
+ * way PreviewView's FILL_CENTER does.
+ */
+public class DetectionOverlay extends View {
+
+    /** Keep the rectangle this long after the card was last seen, to avoid flicker. */
+    private static final long HIDE_AFTER_MS = 300;
+
+    private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG);
+    private final float radius;
+    private final RectF drawRect = new RectF();
+
+    @Nullable
+    private RectF box;
+    private int frameWidth;
+    private int frameHeight;
+    private boolean mirror;
+    private long lastBoxAt;
+
+    public DetectionOverlay(Context context) {
+        super(context);
+        float density = context.getResources().getDisplayMetrics().density;
+        radius = 8 * density;
+        paint.setColor(Color.RED);
+        paint.setStyle(Paint.Style.STROKE);
+        paint.setStrokeWidth(3 * density);
+    }
+
+    public void setMirror(boolean mirror) {
+        this.mirror = mirror;
+    }
+
+    /** Must be called on the UI thread. */
+    public void setBox(@Nullable RectF box, int frameWidth, int frameHeight) {
+        long now = SystemClock.uptimeMillis();
+        if (box == null) {
+            if (this.box == null || now - lastBoxAt < HIDE_AFTER_MS) return;
+        } else {
+            lastBoxAt = now;
+        }
+        this.box = box;
+        this.frameWidth = frameWidth;
+        this.frameHeight = frameHeight;
+        invalidate();
+    }
+
+    public void clear() {
+        box = null;
+        invalidate();
+    }
+
+    @Override
+    protected void onDraw(Canvas canvas) {
+        if (box == null || frameWidth == 0 || frameHeight == 0) return;
+
+        float vw = getWidth();
+        float vh = getHeight();
+        float scale = Math.max(vw / frameWidth, vh / frameHeight);
+        float sw = frameWidth * scale;
+        float sh = frameHeight * scale;
+        float dx = (vw - sw) / 2f;
+        float dy = (vh - sh) / 2f;
+
+        float left = mirror ? 1f - box.right : box.left;
+        float right = mirror ? 1f - box.left : box.right;
+        drawRect.set(dx + left * sw, dy + box.top * sh, dx + right * sw, dy + box.bottom * sh);
+        canvas.drawRoundRect(drawRect, radius, radius, paint);
+    }
+}