capacitor-card-scanner 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -27
- package/android/src/main/java/pt/tcgclassifier/cardscanner/CardDetector.java +523 -418
- package/android/src/main/java/pt/tcgclassifier/cardscanner/CardScannerPlugin.java +191 -189
- package/dist/docs.json +45 -14
- package/dist/esm/definitions.d.ts +21 -1
- package/dist/esm/definitions.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,418 +1,523 @@
|
|
|
1
|
-
package pt.tcgclassifier.cardscanner;
|
|
2
|
-
|
|
3
|
-
import android.graphics.Bitmap;
|
|
4
|
-
import android.graphics.
|
|
5
|
-
import android.graphics.
|
|
6
|
-
import android.graphics.
|
|
7
|
-
import android.
|
|
8
|
-
import android.
|
|
9
|
-
import
|
|
10
|
-
import androidx.annotation.
|
|
11
|
-
import androidx.
|
|
12
|
-
import androidx.camera.core.
|
|
13
|
-
import
|
|
14
|
-
import com.getcapacitor.
|
|
15
|
-
import com.getcapacitor.
|
|
16
|
-
import com.
|
|
17
|
-
import com.google.
|
|
18
|
-
import com.google.mlkit.vision.
|
|
19
|
-
import com.google.mlkit.vision.objects.
|
|
20
|
-
import com.google.mlkit.vision.objects.
|
|
21
|
-
import com.google.mlkit.vision.objects.
|
|
22
|
-
import com.google.mlkit.vision.
|
|
23
|
-
import com.google.mlkit.vision.text.
|
|
24
|
-
import com.google.mlkit.vision.text.
|
|
25
|
-
import com.google.mlkit.vision.text.
|
|
26
|
-
import
|
|
27
|
-
import java.
|
|
28
|
-
import java.
|
|
29
|
-
import java.util.
|
|
30
|
-
import java.util.
|
|
31
|
-
import java.util.
|
|
32
|
-
import java.util.
|
|
33
|
-
import java.util.
|
|
34
|
-
import java.util.regex.
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*/
|
|
78
|
-
public
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
private final
|
|
135
|
-
|
|
136
|
-
private final
|
|
137
|
-
|
|
138
|
-
private
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
private
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
private
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
public
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
String
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
if (
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
private
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
1
|
+
package pt.tcgclassifier.cardscanner;
|
|
2
|
+
|
|
3
|
+
import android.graphics.Bitmap;
|
|
4
|
+
import android.graphics.Color;
|
|
5
|
+
import android.graphics.Matrix;
|
|
6
|
+
import android.graphics.Rect;
|
|
7
|
+
import android.graphics.RectF;
|
|
8
|
+
import android.os.SystemClock;
|
|
9
|
+
import android.util.Base64;
|
|
10
|
+
import androidx.annotation.NonNull;
|
|
11
|
+
import androidx.annotation.Nullable;
|
|
12
|
+
import androidx.camera.core.ImageAnalysis;
|
|
13
|
+
import androidx.camera.core.ImageProxy;
|
|
14
|
+
import com.getcapacitor.JSArray;
|
|
15
|
+
import com.getcapacitor.JSObject;
|
|
16
|
+
import com.getcapacitor.Logger;
|
|
17
|
+
import com.google.android.gms.tasks.Tasks;
|
|
18
|
+
import com.google.mlkit.vision.common.InputImage;
|
|
19
|
+
import com.google.mlkit.vision.objects.DetectedObject;
|
|
20
|
+
import com.google.mlkit.vision.objects.ObjectDetection;
|
|
21
|
+
import com.google.mlkit.vision.objects.ObjectDetector;
|
|
22
|
+
import com.google.mlkit.vision.objects.defaults.ObjectDetectorOptions;
|
|
23
|
+
import com.google.mlkit.vision.text.Text;
|
|
24
|
+
import com.google.mlkit.vision.text.TextRecognition;
|
|
25
|
+
import com.google.mlkit.vision.text.TextRecognizer;
|
|
26
|
+
import com.google.mlkit.vision.text.latin.TextRecognizerOptions;
|
|
27
|
+
import java.io.ByteArrayOutputStream;
|
|
28
|
+
import java.text.Normalizer;
|
|
29
|
+
import java.util.ArrayList;
|
|
30
|
+
import java.util.Arrays;
|
|
31
|
+
import java.util.Comparator;
|
|
32
|
+
import java.util.List;
|
|
33
|
+
import java.util.Locale;
|
|
34
|
+
import java.util.regex.Matcher;
|
|
35
|
+
import java.util.regex.Pattern;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Locates the card in every camera frame (for the on-screen rectangle) and,
|
|
39
|
+
* every {@code intervalMs}, runs OCR inside it. A card is reported once the
|
|
40
|
+
* same collector number (or name) has been read on several consecutive runs.
|
|
41
|
+
*/
|
|
42
|
+
public class CardDetector implements ImageAnalysis.Analyzer {
|
|
43
|
+
|
|
44
|
+
public interface Listener {
|
|
45
|
+
void onCardDetected(JSObject result);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Tuning knobs, all overridable from JS through startDetection(). */
|
|
49
|
+
public static class Options {
|
|
50
|
+
|
|
51
|
+
/** Minimum time between two OCR runs. */
|
|
52
|
+
public long intervalMs = 300;
|
|
53
|
+
/** Consecutive OCR runs that must agree before a card is reported. */
|
|
54
|
+
public int stableFrames = 2;
|
|
55
|
+
/** Include a JPEG of the card in the event. */
|
|
56
|
+
public boolean includeImage = false;
|
|
57
|
+
/** Expected width / height of an upright card (63 x 88 mm). */
|
|
58
|
+
public float cardAspectRatio = 63f / 88f;
|
|
59
|
+
/** Allowed relative deviation from cardAspectRatio (0.15 = +/-15%). */
|
|
60
|
+
public float aspectTolerance = 0.15f;
|
|
61
|
+
/** Smallest card size, as a fraction of the frame area. */
|
|
62
|
+
public float minCardArea = 0.08f;
|
|
63
|
+
/** Largest card size, as a fraction of the frame area. */
|
|
64
|
+
public float maxCardArea = 0.9f;
|
|
65
|
+
/** Only run OCR when a card-shaped object is visible. */
|
|
66
|
+
public boolean requireCard = true;
|
|
67
|
+
/** Label every detected object with its area/ratio and the filter that rejected it. */
|
|
68
|
+
public boolean debug = false;
|
|
69
|
+
/** Outline (in grey) the objects that were not accepted as a card. */
|
|
70
|
+
public boolean showRejected = true;
|
|
71
|
+
/** How long the card must be out of view before the same card can be reported again. */
|
|
72
|
+
public long repeatAfterMs = 1500;
|
|
73
|
+
/**
|
|
74
|
+
* While the card outline stays in view (gaps shorter than this are
|
|
75
|
+
* tolerated), at most one card is reported, even if later readings
|
|
76
|
+
* give a different name. A longer gap means a new card is in view.
|
|
77
|
+
*/
|
|
78
|
+
public long cardGapMs = 500;
|
|
79
|
+
/**
|
|
80
|
+
* Two card images whose perceptual hashes differ in at most this many
|
|
81
|
+
* bits (of 64) are the same card. Lets a card slid over another one be
|
|
82
|
+
* reported, and a card that stays in view be reported only once.
|
|
83
|
+
*/
|
|
84
|
+
public int sameCardMaxDistance = 16;
|
|
85
|
+
/** The name is looked for only in the first N lines of the card (top to bottom). */
|
|
86
|
+
public int nameLines = 3;
|
|
87
|
+
/**
|
|
88
|
+
* Words/phrases that are never the card name (card types, subtypes...).
|
|
89
|
+
* A line made only of these words is skipped; "Energy Retrieval" is still a name.
|
|
90
|
+
*/
|
|
91
|
+
public List<String> reservedWords = Arrays.asList(
|
|
92
|
+
"Basic", "Stage", "Stage 1", "Stage 2", "Trainer", "Item", "Supporter", "Stadium",
|
|
93
|
+
"Energy", "Special Energy", "Basic Energy", "Pokémon", "Pokémon Tool", "Tera"
|
|
94
|
+
);
|
|
95
|
+
/** Lines starting with any of these are never the card name. */
|
|
96
|
+
public List<String> reservedPrefixes = Arrays.asList("Evolves from", "HP");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** An object found by ML Kit, with the values the card filters were checked against. */
|
|
100
|
+
public static class Candidate {
|
|
101
|
+
|
|
102
|
+
/** Bounds normalized (0-1) to the upright frame. */
|
|
103
|
+
public final RectF box;
|
|
104
|
+
/** Fraction of the frame area. */
|
|
105
|
+
public final float area;
|
|
106
|
+
/** Width / height. */
|
|
107
|
+
public final float ratio;
|
|
108
|
+
/** Null when accepted, otherwise the filter that rejected it ("area" or "ratio"). */
|
|
109
|
+
@Nullable
|
|
110
|
+
public final String rejectedBy;
|
|
111
|
+
|
|
112
|
+
Candidate(RectF box, float area, float ratio, @Nullable String rejectedBy) {
|
|
113
|
+
this.box = box;
|
|
114
|
+
this.area = area;
|
|
115
|
+
this.ratio = ratio;
|
|
116
|
+
this.rejectedBy = rejectedBy;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
public String label() {
|
|
120
|
+
String s = String.format(Locale.US, "a=%.2f r=%.2f", area, ratio);
|
|
121
|
+
return rejectedBy == null ? s : s + " x" + rejectedBy;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
public interface BoxListener {
|
|
126
|
+
/**
|
|
127
|
+
* @param card card bounds normalized (0-1) to the upright frame, or null when no card is visible
|
|
128
|
+
* @param candidates every detected object, accepted or not
|
|
129
|
+
*/
|
|
130
|
+
void onDetections(@Nullable RectF card, List<Candidate> candidates, int frameWidth, int frameHeight);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// e.g. "025/165", "SV001/SV122", "TG05/TG30", "GG01/GG70"
|
|
134
|
+
private static final Pattern NUMBER = Pattern.compile("\\b([A-Za-z]{0,4}\\d{1,3}[a-z]?)\\s*/\\s*([A-Za-z]{0,4}\\d{1,3})\\b");
|
|
135
|
+
private static final Pattern HP_SUFFIX = Pattern.compile("\\s*HP\\s*\\d+\\s*$", Pattern.CASE_INSENSITIVE);
|
|
136
|
+
private static final Pattern LETTER = Pattern.compile("\\p{L}");
|
|
137
|
+
/** Names at least this similar are treated as the same card. */
|
|
138
|
+
private static final float SAME_NAME_SIMILARITY = 0.8f;
|
|
139
|
+
/** Extra margin around the detected card before running OCR. */
|
|
140
|
+
private static final float CROP_MARGIN = 0.04f;
|
|
141
|
+
|
|
142
|
+
private final ObjectDetector objectDetector = ObjectDetection.getClient(
|
|
143
|
+
new ObjectDetectorOptions.Builder().setDetectorMode(ObjectDetectorOptions.STREAM_MODE).build()
|
|
144
|
+
);
|
|
145
|
+
private final TextRecognizer recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS);
|
|
146
|
+
private final Options options;
|
|
147
|
+
private final Listener listener;
|
|
148
|
+
/** reservedWords / reservedPrefixes, normalized; words longest first. */
|
|
149
|
+
private final List<String> reservedWords = new ArrayList<>();
|
|
150
|
+
private final List<String> reservedPrefixes = new ArrayList<>();
|
|
151
|
+
private volatile BoxListener boxListener;
|
|
152
|
+
|
|
153
|
+
// Only touched from the analyzer thread.
|
|
154
|
+
private long lastOcrAt;
|
|
155
|
+
private long lastSeenAt;
|
|
156
|
+
/** Last frame with a card outline, and whether that card was already reported. */
|
|
157
|
+
private long lastCardAt;
|
|
158
|
+
private boolean reportedInView;
|
|
159
|
+
private String candidateKey;
|
|
160
|
+
private int candidateCount;
|
|
161
|
+
// Last reported card. A reading is the same card if either its name or its
|
|
162
|
+
// number matches, because OCR does not read the number on every frame.
|
|
163
|
+
private String lastEmittedName;
|
|
164
|
+
private String lastEmittedNumber;
|
|
165
|
+
/** Perceptual hash of the last reported card image, or null. */
|
|
166
|
+
private Long lastEmittedHash;
|
|
167
|
+
|
|
168
|
+
public CardDetector(Options options, Listener listener) {
|
|
169
|
+
this.options = options;
|
|
170
|
+
this.options.stableFrames = Math.max(1, options.stableFrames);
|
|
171
|
+
this.options.nameLines = Math.max(1, options.nameLines);
|
|
172
|
+
this.listener = listener;
|
|
173
|
+
for (String w : options.reservedWords) {
|
|
174
|
+
String n = normalize(w);
|
|
175
|
+
if (!n.isEmpty()) reservedWords.add(n);
|
|
176
|
+
}
|
|
177
|
+
// Longest first, so "Pokémon Tool" is removed before "Pokémon".
|
|
178
|
+
reservedWords.sort(Comparator.comparingInt(String::length).reversed());
|
|
179
|
+
for (String p : options.reservedPrefixes) {
|
|
180
|
+
String n = normalize(p);
|
|
181
|
+
if (!n.isEmpty()) reservedPrefixes.add(n);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
public boolean isDebug() {
|
|
186
|
+
return options.debug;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
public boolean isShowRejected() {
|
|
190
|
+
return options.showRejected;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
public void setBoxListener(BoxListener boxListener) {
|
|
194
|
+
this.boxListener = boxListener;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
public void close() {
|
|
198
|
+
objectDetector.close();
|
|
199
|
+
recognizer.close();
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
@Override
|
|
203
|
+
public void analyze(@NonNull ImageProxy image) {
|
|
204
|
+
long now = SystemClock.elapsedRealtime();
|
|
205
|
+
Bitmap upright;
|
|
206
|
+
try {
|
|
207
|
+
// Rotate once, so both detectors and all coordinates share the same upright frame.
|
|
208
|
+
upright = rotate(image.toBitmap(), image.getImageInfo().getRotationDegrees());
|
|
209
|
+
} finally {
|
|
210
|
+
image.close();
|
|
211
|
+
}
|
|
212
|
+
int frameW = upright.getWidth();
|
|
213
|
+
int frameH = upright.getHeight();
|
|
214
|
+
|
|
215
|
+
try {
|
|
216
|
+
List<DetectedObject> objects = Tasks.await(objectDetector.process(InputImage.fromBitmap(upright, 0)));
|
|
217
|
+
List<Candidate> candidates = new ArrayList<>();
|
|
218
|
+
Rect card = pickCard(objects, frameW, frameH, candidates);
|
|
219
|
+
RectF cardBox = card == null ? null : normalize(card, frameW, frameH);
|
|
220
|
+
// The card counts as "in view" while its outline is visible, even if
|
|
221
|
+
// its text is not readable on this frame.
|
|
222
|
+
if (card != null) {
|
|
223
|
+
// A gap in the outline means the card left (or was swapped).
|
|
224
|
+
if (now - lastCardAt > options.cardGapMs) reportedInView = false;
|
|
225
|
+
lastCardAt = now;
|
|
226
|
+
lastSeenAt = now;
|
|
227
|
+
}
|
|
228
|
+
BoxListener bl = boxListener;
|
|
229
|
+
if (bl != null) bl.onDetections(cardBox, candidates, frameW, frameH);
|
|
230
|
+
|
|
231
|
+
if (now - lastOcrAt < options.intervalMs) return;
|
|
232
|
+
lastOcrAt = now;
|
|
233
|
+
|
|
234
|
+
if (card == null && options.requireCard) {
|
|
235
|
+
noCardRead(now);
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
Bitmap region = card == null ? upright : crop(upright, card);
|
|
240
|
+
Text text = Tasks.await(recognizer.process(InputImage.fromBitmap(region, 0)));
|
|
241
|
+
handle(text, region, cardBox, now);
|
|
242
|
+
} catch (Exception e) {
|
|
243
|
+
Logger.error("CardDetector", "Detection failed", e);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* The biggest detected object with the size and upright proportions of a card.
|
|
249
|
+
* Every object, accepted or not, is added to {@code candidates}.
|
|
250
|
+
*/
|
|
251
|
+
@Nullable
|
|
252
|
+
private Rect pickCard(List<DetectedObject> objects, int frameW, int frameH, List<Candidate> candidates) {
|
|
253
|
+
float frameArea = (float) frameW * frameH;
|
|
254
|
+
float minRatio = options.cardAspectRatio * (1f - options.aspectTolerance);
|
|
255
|
+
float maxRatio = options.cardAspectRatio * (1f + options.aspectTolerance);
|
|
256
|
+
Rect best = null;
|
|
257
|
+
long bestArea = 0;
|
|
258
|
+
for (DetectedObject o : objects) {
|
|
259
|
+
Rect r = new Rect(o.getBoundingBox());
|
|
260
|
+
if (!r.intersect(0, 0, frameW, frameH) || r.height() == 0) continue;
|
|
261
|
+
|
|
262
|
+
long area = (long) r.width() * r.height();
|
|
263
|
+
float fraction = area / frameArea;
|
|
264
|
+
float ratio = r.width() / (float) r.height();
|
|
265
|
+
String rejectedBy = null;
|
|
266
|
+
if (fraction < options.minCardArea || fraction > options.maxCardArea) {
|
|
267
|
+
rejectedBy = "area";
|
|
268
|
+
} else if (ratio < minRatio || ratio > maxRatio) {
|
|
269
|
+
rejectedBy = "ratio";
|
|
270
|
+
}
|
|
271
|
+
candidates.add(new Candidate(normalize(r, frameW, frameH), fraction, ratio, rejectedBy));
|
|
272
|
+
if (rejectedBy != null) continue;
|
|
273
|
+
|
|
274
|
+
if (area > bestArea) {
|
|
275
|
+
best = r;
|
|
276
|
+
bestArea = area;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return best;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
private static RectF normalize(Rect r, int frameW, int frameH) {
|
|
283
|
+
return new RectF(r.left / (float) frameW, r.top / (float) frameH, r.right / (float) frameW, r.bottom / (float) frameH);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
private static Bitmap rotate(Bitmap bitmap, int rotation) {
|
|
287
|
+
if (rotation == 0) return bitmap;
|
|
288
|
+
Matrix matrix = new Matrix();
|
|
289
|
+
matrix.postRotate(rotation);
|
|
290
|
+
return Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
private static Bitmap crop(Bitmap bitmap, Rect card) {
|
|
294
|
+
int mx = Math.round(card.width() * CROP_MARGIN);
|
|
295
|
+
int my = Math.round(card.height() * CROP_MARGIN);
|
|
296
|
+
int left = Math.max(0, card.left - mx);
|
|
297
|
+
int top = Math.max(0, card.top - my);
|
|
298
|
+
int right = Math.min(bitmap.getWidth(), card.right + mx);
|
|
299
|
+
int bottom = Math.min(bitmap.getHeight(), card.bottom + my);
|
|
300
|
+
return Bitmap.createBitmap(bitmap, left, top, right - left, bottom - top);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
private void handle(Text text, Bitmap region, @Nullable RectF cardBox, long now) {
|
|
304
|
+
float w = region.getWidth();
|
|
305
|
+
float h = region.getHeight();
|
|
306
|
+
JSArray lines = new JSArray();
|
|
307
|
+
String number = null;
|
|
308
|
+
List<Text.Line> ordered = new ArrayList<>();
|
|
309
|
+
|
|
310
|
+
for (Text.TextBlock block : text.getTextBlocks()) {
|
|
311
|
+
for (Text.Line line : block.getLines()) {
|
|
312
|
+
String s = line.getText().trim();
|
|
313
|
+
Rect r = line.getBoundingBox();
|
|
314
|
+
if (s.isEmpty() || r == null) continue;
|
|
315
|
+
|
|
316
|
+
JSObject item = new JSObject();
|
|
317
|
+
item.put("text", s);
|
|
318
|
+
item.put("box", box(r.left / w, r.top / h, r.width() / w, r.height() / h));
|
|
319
|
+
lines.put(item);
|
|
320
|
+
|
|
321
|
+
if (number == null) {
|
|
322
|
+
Matcher m = NUMBER.matcher(s);
|
|
323
|
+
if (m.find()) number = m.group(1) + "/" + m.group(2);
|
|
324
|
+
}
|
|
325
|
+
ordered.add(line);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
String name = findName(ordered);
|
|
329
|
+
|
|
330
|
+
String key = number != null ? number : name;
|
|
331
|
+
if (key == null) {
|
|
332
|
+
noCardRead(now);
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
lastSeenAt = now;
|
|
336
|
+
|
|
337
|
+
if (key.equals(candidateKey)) {
|
|
338
|
+
candidateCount++;
|
|
339
|
+
} else {
|
|
340
|
+
candidateKey = key;
|
|
341
|
+
candidateCount = 1;
|
|
342
|
+
}
|
|
343
|
+
if (candidateCount < options.stableFrames) return;
|
|
344
|
+
long hash = imageHash(region);
|
|
345
|
+
if (isSameCardAsLast(name, number, hash, cardBox != null)) return;
|
|
346
|
+
reportedInView = cardBox != null;
|
|
347
|
+
lastEmittedName = name;
|
|
348
|
+
lastEmittedNumber = number;
|
|
349
|
+
lastEmittedHash = hash;
|
|
350
|
+
|
|
351
|
+
JSObject result = new JSObject();
|
|
352
|
+
result.put("name", name);
|
|
353
|
+
result.put("collectorNumber", number);
|
|
354
|
+
result.put("imageHash", String.format(Locale.ROOT, "%016x", hash));
|
|
355
|
+
result.put("text", text.getText());
|
|
356
|
+
result.put("lines", lines);
|
|
357
|
+
if (cardBox != null) {
|
|
358
|
+
result.put("cardBox", box(cardBox.left, cardBox.top, cardBox.width(), cardBox.height()));
|
|
359
|
+
}
|
|
360
|
+
if (options.includeImage) {
|
|
361
|
+
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
|
362
|
+
region.compress(Bitmap.CompressFormat.JPEG, 80, out);
|
|
363
|
+
result.put("image", Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP));
|
|
364
|
+
}
|
|
365
|
+
listener.onCardDetected(result);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
private void noCardRead(long now) {
|
|
369
|
+
candidateKey = null;
|
|
370
|
+
candidateCount = 0;
|
|
371
|
+
if (now - lastSeenAt > options.repeatAfterMs) {
|
|
372
|
+
lastEmittedName = null;
|
|
373
|
+
lastEmittedNumber = null;
|
|
374
|
+
lastEmittedHash = null;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Whether this reading is the card that was just reported. The image hash
|
|
380
|
+
* decides first; the text (number, then name) guards against glare making
|
|
381
|
+
* the same card look different.
|
|
382
|
+
*/
|
|
383
|
+
private boolean isSameCardAsLast(String name, String number, long hash, boolean cardInView) {
|
|
384
|
+
if (lastEmittedHash == null) {
|
|
385
|
+
// Nothing to compare the picture with: one event per card in view.
|
|
386
|
+
return isLastEmitted(name, number) || (cardInView && reportedInView);
|
|
387
|
+
}
|
|
388
|
+
if (Long.bitCount(hash ^ lastEmittedHash) <= options.sameCardMaxDistance) return true;
|
|
389
|
+
|
|
390
|
+
// The picture changed: a new card, unless the text says it is the same one.
|
|
391
|
+
if (number != null && lastEmittedNumber != null) return number.equals(lastEmittedNumber);
|
|
392
|
+
return isLastEmitted(name, number);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* 64-bit difference hash: the card split into 9x8 cells, each cell's average
|
|
397
|
+
* grey level, one bit per "is this cell brighter than its right neighbour".
|
|
398
|
+
* Averaging matters: sampling single pixels makes the hash unstable. On real
|
|
399
|
+
* photos the same card differed by 9-13 bits and different cards by 32+.
|
|
400
|
+
*/
|
|
401
|
+
private static long imageHash(Bitmap bitmap) {
|
|
402
|
+
int w = bitmap.getWidth();
|
|
403
|
+
int h = bitmap.getHeight();
|
|
404
|
+
int[] pixels = new int[w * h];
|
|
405
|
+
bitmap.getPixels(pixels, 0, w, 0, 0, w, h);
|
|
406
|
+
// Sampling ~100 points per side is plenty for 9x8 averages.
|
|
407
|
+
int step = Math.max(1, Math.min(w, h) / 100);
|
|
408
|
+
|
|
409
|
+
float[][] cells = new float[8][9];
|
|
410
|
+
for (int cy = 0; cy < 8; cy++) {
|
|
411
|
+
int y0 = cy * h / 8;
|
|
412
|
+
int y1 = (cy + 1) * h / 8;
|
|
413
|
+
for (int cx = 0; cx < 9; cx++) {
|
|
414
|
+
int x0 = cx * w / 9;
|
|
415
|
+
int x1 = (cx + 1) * w / 9;
|
|
416
|
+
long sum = 0;
|
|
417
|
+
int count = 0;
|
|
418
|
+
for (int y = y0; y < y1; y += step) {
|
|
419
|
+
for (int x = x0; x < x1; x += step) {
|
|
420
|
+
sum += grey(pixels[y * w + x]);
|
|
421
|
+
count++;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
cells[cy][cx] = count == 0 ? 0 : (float) sum / count;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
long hash = 0;
|
|
429
|
+
for (int y = 0; y < 8; y++) {
|
|
430
|
+
for (int x = 0; x < 8; x++) {
|
|
431
|
+
hash <<= 1;
|
|
432
|
+
if (cells[y][x] > cells[y][x + 1]) hash |= 1;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
return hash;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
private static int grey(int color) {
|
|
439
|
+
return (Color.red(color) * 299 + Color.green(color) * 587 + Color.blue(color) * 114) / 1000;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
private boolean isLastEmitted(String name, String number) {
|
|
443
|
+
if (number != null && number.equals(lastEmittedNumber)) return true;
|
|
444
|
+
return name != null && lastEmittedName != null && similarity(name, lastEmittedName) >= SAME_NAME_SIMILARITY;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** 0-1 similarity (LCS based), so an OCR slip like "Evolutlon" still matches "Evolution". */
|
|
448
|
+
private static float similarity(String a, String b) {
|
|
449
|
+
a = a.toLowerCase(Locale.ROOT);
|
|
450
|
+
b = b.toLowerCase(Locale.ROOT);
|
|
451
|
+
if (a.equals(b)) return 1f;
|
|
452
|
+
int[] prev = new int[b.length() + 1];
|
|
453
|
+
int[] cur = new int[b.length() + 1];
|
|
454
|
+
for (int i = 1; i <= a.length(); i++) {
|
|
455
|
+
for (int j = 1; j <= b.length(); j++) {
|
|
456
|
+
cur[j] = a.charAt(i - 1) == b.charAt(j - 1) ? prev[j - 1] + 1 : Math.max(prev[j], cur[j - 1]);
|
|
457
|
+
}
|
|
458
|
+
int[] tmp = prev;
|
|
459
|
+
prev = cur;
|
|
460
|
+
cur = tmp;
|
|
461
|
+
}
|
|
462
|
+
return 2f * prev[b.length()] / (a.length() + b.length());
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
private static JSObject box(float x, float y, float width, float height) {
|
|
466
|
+
JSObject box = new JSObject();
|
|
467
|
+
box.put("x", x);
|
|
468
|
+
box.put("y", y);
|
|
469
|
+
box.put("width", width);
|
|
470
|
+
box.put("height", height);
|
|
471
|
+
return box;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** First line, among the top {@code nameLines}, that is not made of reserved words. */
|
|
475
|
+
@Nullable
|
|
476
|
+
private String findName(List<Text.Line> lines) {
|
|
477
|
+
lines.sort(Comparator.comparingInt(l -> l.getBoundingBox().top));
|
|
478
|
+
for (int i = 0; i < lines.size() && i < options.nameLines; i++) {
|
|
479
|
+
String s = HP_SUFFIX.matcher(lines.get(i).getText().trim()).replaceAll("").trim();
|
|
480
|
+
if (!isReserved(s) && countLetters(s) >= 3) return s;
|
|
481
|
+
}
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
private boolean isReserved(String line) {
|
|
486
|
+
String normalized = normalize(line);
|
|
487
|
+
|
|
488
|
+
// Printed labels are often read without spaces ("STAGE1"), so check both ways.
|
|
489
|
+
String compact = normalized.replace(" ", "");
|
|
490
|
+
for (String w : reservedWords) {
|
|
491
|
+
compact = compact.replace(w.replace(" ", ""), "");
|
|
492
|
+
}
|
|
493
|
+
if (countLetters(compact) < 3) return true;
|
|
494
|
+
|
|
495
|
+
String rest = " " + normalized + " ";
|
|
496
|
+
for (String w : reservedWords) {
|
|
497
|
+
rest = rest.replace(" " + w + " ", " ");
|
|
498
|
+
}
|
|
499
|
+
rest = rest.trim();
|
|
500
|
+
if (countLetters(rest) < 3) return true;
|
|
501
|
+
// Checked after removing reserved words, so "STAGE 1 Evolves from Pichu" is caught too.
|
|
502
|
+
String restCompact = rest.replace(" ", "");
|
|
503
|
+
for (String p : reservedPrefixes) {
|
|
504
|
+
if (rest.equals(p) || rest.startsWith(p + " ")) return true;
|
|
505
|
+
// Multi-word prefixes read without spaces ("Evolvesfrom Pichu").
|
|
506
|
+
if (p.contains(" ") && restCompact.startsWith(p.replace(" ", ""))) return true;
|
|
507
|
+
}
|
|
508
|
+
return false;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/** Upper case, no accents, only letters/digits separated by single spaces. */
|
|
512
|
+
private static String normalize(String s) {
|
|
513
|
+
String n = Normalizer.normalize(s, Normalizer.Form.NFD).replaceAll("\\p{M}", "");
|
|
514
|
+
return n.toUpperCase(Locale.ROOT).replaceAll("[^\\p{L}\\p{Nd}]+", " ").trim();
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
private static int countLetters(String s) {
|
|
518
|
+
Matcher m = LETTER.matcher(s);
|
|
519
|
+
int letters = 0;
|
|
520
|
+
while (m.find()) letters++;
|
|
521
|
+
return letters;
|
|
522
|
+
}
|
|
523
|
+
}
|