@pictor-sdk/video-android 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +40 -0
  3. package/android/CMakeLists.txt +30 -0
  4. package/android/build.gradle +119 -0
  5. package/android/src/main/AndroidManifest.xml +1 -0
  6. package/android/src/main/cpp/pictor_overlaystage.cpp +372 -0
  7. package/android/src/main/cpp/pictor_publish.cpp +87 -0
  8. package/android/src/main/cpp/pictor_videosurface.cpp +735 -0
  9. package/android/src/main/java/dev/pictor/video/AacPriming.kt +261 -0
  10. package/android/src/main/java/dev/pictor/video/AudioMixer.kt +1112 -0
  11. package/android/src/main/java/dev/pictor/video/AudioProbe.kt +162 -0
  12. package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt +337 -0
  13. package/android/src/main/java/dev/pictor/video/CodecLedger.kt +126 -0
  14. package/android/src/main/java/dev/pictor/video/CompositionSpike.kt +978 -0
  15. package/android/src/main/java/dev/pictor/video/Diagnostics.kt +54 -0
  16. package/android/src/main/java/dev/pictor/video/EncoderGate.kt +476 -0
  17. package/android/src/main/java/dev/pictor/video/ExportGate.kt +1974 -0
  18. package/android/src/main/java/dev/pictor/video/ExportHeartbeat.kt +72 -0
  19. package/android/src/main/java/dev/pictor/video/ExportProgress.kt +103 -0
  20. package/android/src/main/java/dev/pictor/video/FrameClock.kt +591 -0
  21. package/android/src/main/java/dev/pictor/video/FramePending.kt +181 -0
  22. package/android/src/main/java/dev/pictor/video/GateAudio.kt +456 -0
  23. package/android/src/main/java/dev/pictor/video/GateDestination.kt +445 -0
  24. package/android/src/main/java/dev/pictor/video/GateFrames.kt +662 -0
  25. package/android/src/main/java/dev/pictor/video/GateMux.kt +185 -0
  26. package/android/src/main/java/dev/pictor/video/ManagedSources.kt +333 -0
  27. package/android/src/main/java/dev/pictor/video/Media3Player.kt +934 -0
  28. package/android/src/main/java/dev/pictor/video/MediaStoreSave.kt +419 -0
  29. package/android/src/main/java/dev/pictor/video/OverlayResources.kt +169 -0
  30. package/android/src/main/java/dev/pictor/video/PictorVideoModule.kt +1612 -0
  31. package/android/src/main/java/dev/pictor/video/PictorVideoPackage.kt +26 -0
  32. package/android/src/main/java/dev/pictor/video/PlaybackAnchor.kt +161 -0
  33. package/android/src/main/java/dev/pictor/video/PreviewAudioPlayer.kt +291 -0
  34. package/android/src/main/java/dev/pictor/video/PreviewConfig.kt +96 -0
  35. package/android/src/main/java/dev/pictor/video/PreviewMix.kt +396 -0
  36. package/android/src/main/java/dev/pictor/video/PreviewSync.kt +374 -0
  37. package/android/src/main/java/dev/pictor/video/PreviewTrace.kt +478 -0
  38. package/android/src/main/java/dev/pictor/video/PrimingTrim.kt +337 -0
  39. package/android/src/main/java/dev/pictor/video/RepeatingEncoderFactory.kt +97 -0
  40. package/android/src/main/java/dev/pictor/video/SceneProgram.kt +169 -0
  41. package/android/src/main/java/dev/pictor/video/SceneProgramNative.kt +299 -0
  42. package/android/src/main/java/dev/pictor/video/SkiaPassThroughEffect.kt +695 -0
  43. package/android/src/main/java/dev/pictor/video/SurfaceAttachment.kt +110 -0
  44. package/android/src/main/java/dev/pictor/video/ThumbnailDecoder.kt +171 -0
  45. package/android/src/main/java/dev/pictor/video/VideoSurface.kt +394 -0
  46. package/app.plugin.js +43 -0
  47. package/dist/index.d.mts +3622 -0
  48. package/dist/index.d.ts +3622 -0
  49. package/dist/index.js +3737 -0
  50. package/dist/index.js.map +1 -0
  51. package/dist/index.mjs +3625 -0
  52. package/dist/index.mjs.map +1 -0
  53. package/package.json +82 -0
  54. package/plugin/gradleProperties.d.ts +19 -0
  55. package/plugin/gradleProperties.js +64 -0
  56. package/react-native.config.js +13 -0
  57. package/src/NativePictorVideo.ts +574 -0
  58. package/src/audioProbe.ts +72 -0
  59. package/src/compositionSpike.ts +305 -0
  60. package/src/convergence.ts +112 -0
  61. package/src/encoderGate.ts +66 -0
  62. package/src/exportDestination.ts +149 -0
  63. package/src/exportJob.ts +232 -0
  64. package/src/exportProgress.ts +391 -0
  65. package/src/exportProgressPump.ts +190 -0
  66. package/src/exporter.ts +655 -0
  67. package/src/index.ts +276 -0
  68. package/src/managedSources.ts +252 -0
  69. package/src/media3.ts +412 -0
  70. package/src/mediaStore.ts +177 -0
  71. package/src/memoryGuard.ts +118 -0
  72. package/src/mixSources.ts +82 -0
  73. package/src/overlayStaging.ts +216 -0
  74. package/src/playbackLifecycle.ts +137 -0
  75. package/src/player.ts +39 -0
  76. package/src/playlistSession.ts +125 -0
  77. package/src/previewAudioDiagnostics.ts +67 -0
  78. package/src/previewAudioSession.ts +168 -0
  79. package/src/previewCadence.ts +950 -0
  80. package/src/previewMix.ts +212 -0
  81. package/src/previewTrace.ts +348 -0
  82. package/src/spikeExcursions.ts +405 -0
  83. package/src/submissionScoring.ts +680 -0
  84. package/src/surfaceOwner.ts +264 -0
  85. package/src/surfaceRecovery.ts +121 -0
  86. package/src/syncTrace.ts +189 -0
  87. package/src/thumbnailProvider.ts +163 -0
  88. package/src/tracePhases.ts +217 -0
  89. package/src/useCompositionPlayback.ts +327 -0
  90. package/src/useMedia3Playback.ts +492 -0
  91. package/src/usePreviewAudio.ts +305 -0
  92. package/src/useVideoSurface.ts +282 -0
@@ -0,0 +1,662 @@
1
+ package dev.pictor.video
2
+
3
+ import android.content.Context
4
+ import android.graphics.Bitmap
5
+ import android.media.MediaMetadataRetriever
6
+ import android.net.Uri
7
+ import org.json.JSONArray
8
+ import org.json.JSONObject
9
+ import kotlin.math.abs
10
+
11
+ /**
12
+ * Decoded **display-space** frames, for the gate's pixel floor.
13
+ *
14
+ * `MediaMetadataRetriever` returns frames with the container's rotation already
15
+ * applied, which is the only space in which "does this look the same" is a
16
+ * meaningful question. Comparing stored dimensions instead is how a 1080×1920
17
+ * file carrying a rotation flag and a 1920×1080 file with the orientation baked
18
+ * in look like a difference when they display identically — the exact ambiguity
19
+ * that stalled the first run of this gate.
20
+ *
21
+ * Nothing here encodes or exports. It reads files back and reports numbers.
22
+ */
23
+ internal object GateFrames {
24
+
25
+ /** What one file looks like, in display space. */
26
+ private class Reader(context: Context, uri: String) : AutoCloseable {
27
+ val mmr = MediaMetadataRetriever()
28
+
29
+ init {
30
+ if (uri.startsWith("/")) {
31
+ mmr.setDataSource(uri)
32
+ } else {
33
+ mmr.setDataSource(context, Uri.parse(uri))
34
+ }
35
+ }
36
+
37
+ fun meta(key: Int): String = mmr.extractMetadata(key) ?: ""
38
+
39
+ fun frameAt(us: Long): Bitmap? =
40
+ mmr.getFrameAtTime(us, MediaMetadataRetriever.OPTION_CLOSEST)
41
+
42
+ override fun close() {
43
+ runCatching { mmr.release() }
44
+ }
45
+ }
46
+
47
+ /**
48
+ * Compare two files at `timestampsUs`, in display space.
49
+ *
50
+ * Returns per-timestamp statistics rather than a verdict: the threshold is
51
+ * the caller's to freeze, and freezing it here would be choosing it after
52
+ * seeing the numbers.
53
+ */
54
+ fun compare(
55
+ context: Context,
56
+ leftUri: String,
57
+ rightUri: String,
58
+ timestampsUs: LongArray,
59
+ ): String {
60
+ val out = JSONObject()
61
+ try {
62
+ Reader(context, leftUri).use { left ->
63
+ Reader(context, rightUri).use { right ->
64
+ out.put("leftDisplay", describe(left))
65
+ out.put("rightDisplay", describe(right))
66
+ val samples = JSONArray()
67
+ var worstMax = 0
68
+ var worstMeanTimesThousand = 0L
69
+ for (us in timestampsUs) {
70
+ val a = left.frameAt(us)
71
+ val b = right.frameAt(us)
72
+ val sample = JSONObject().put("timeUs", us)
73
+ if (a == null || b == null) {
74
+ sample.put("error", "no frame")
75
+ } else if (a.width != b.width || a.height != b.height) {
76
+ // A real difference, and not one a tolerance can
77
+ // absorb: the two do not display at the same size.
78
+ sample.put("error", "display size ${a.width}x${a.height} vs ${b.width}x${b.height}")
79
+ } else {
80
+ val stats = diff(a, b)
81
+ sample.put("width", a.width).put("height", a.height)
82
+ sample.put("maxDelta", stats.maxDelta)
83
+ sample.put("meanDelta", stats.meanDelta)
84
+ sample.put("differingPermille", stats.differingPermille)
85
+ if (stats.maxDelta > worstMax) worstMax = stats.maxDelta
86
+ val meanK = (stats.meanDelta * 1000).toLong()
87
+ if (meanK > worstMeanTimesThousand) worstMeanTimesThousand = meanK
88
+ }
89
+ a?.recycle()
90
+ b?.recycle()
91
+ samples.put(sample)
92
+ }
93
+ out.put("samples", samples)
94
+ out.put("worstMaxDelta", worstMax)
95
+ out.put("worstMeanDelta", worstMeanTimesThousand / 1000.0)
96
+ out.put("ok", true)
97
+ }
98
+ }
99
+ } catch (e: Exception) {
100
+ out.put("ok", false).put("error", "${e.javaClass.simpleName}: ${e.message}")
101
+ }
102
+ return out.toString()
103
+ }
104
+
105
+ private fun describe(reader: Reader): JSONObject {
106
+ // Stored dimensions *and* the rotation, so display space can be derived
107
+ // rather than guessed — and a first frame, which is display space by
108
+ // construction.
109
+ val storedW = reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)
110
+ val storedH = reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)
111
+ val rotation = reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)
112
+ val first = reader.frameAt(0)
113
+ val json = JSONObject()
114
+ .put("storedWidth", storedW)
115
+ .put("storedHeight", storedH)
116
+ .put("rotationDegrees", rotation)
117
+ .put("durationMs", reader.meta(MediaMetadataRetriever.METADATA_KEY_DURATION))
118
+ .put("frameCount", reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_FRAME_COUNT))
119
+ .put("colorStandard", reader.meta(MediaMetadataRetriever.METADATA_KEY_COLOR_STANDARD))
120
+ .put("colorRange", reader.meta(MediaMetadataRetriever.METADATA_KEY_COLOR_RANGE))
121
+ .put("colorTransfer", reader.meta(MediaMetadataRetriever.METADATA_KEY_COLOR_TRANSFER))
122
+ .put("hasAudio", reader.meta(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO))
123
+ .put("displayWidth", first?.width ?: 0)
124
+ .put("displayHeight", first?.height ?: 0)
125
+ first?.recycle()
126
+ return json
127
+ }
128
+
129
+ /**
130
+ * Mean colour of a central region, per timestamp, in display space.
131
+ *
132
+ * Deliberately dumb: it reports numbers and draws no conclusion. Which
133
+ * source a colour implies, and which frame of it, is arithmetic the
134
+ * TypeScript oracle owns — native computing that would be exactly the
135
+ * timeline knowledge this architecture keeps out of it.
136
+ *
137
+ * The region avoids the burned-in label, so text pixels cannot pull the
138
+ * mean around.
139
+ */
140
+ fun sample(context: Context, uri: String, timestampsUs: LongArray): String {
141
+ val out = JSONObject()
142
+ try {
143
+ Reader(context, uri).use { reader ->
144
+ out.put("display", describe(reader))
145
+ val samples = JSONArray()
146
+ for (us in timestampsUs) {
147
+ val bitmap = reader.frameAt(us)
148
+ val sample = JSONObject().put("timeUs", us)
149
+ if (bitmap == null) {
150
+ sample.put("error", "no frame")
151
+ } else {
152
+ val w = bitmap.width
153
+ val h = bitmap.height
154
+ // Lower-middle: below the label, inside the field.
155
+ val x0 = w / 4
156
+ val y0 = h / 2
157
+ val cw = w / 2
158
+ val ch = h / 3
159
+ val row = IntArray(cw)
160
+ var r = 0L
161
+ var g = 0L
162
+ var b = 0L
163
+ for (y in y0 until y0 + ch) {
164
+ bitmap.getPixels(row, 0, cw, x0, y, cw, 1)
165
+ for (p in row) {
166
+ r += (p shr 16) and 0xff
167
+ g += (p shr 8) and 0xff
168
+ b += p and 0xff
169
+ }
170
+ }
171
+ val n = cw.toLong() * ch.toLong()
172
+ sample.put("r", r.toDouble() / n)
173
+ sample.put("g", g.toDouble() / n)
174
+ sample.put("b", b.toDouble() / n)
175
+ bitmap.recycle()
176
+ }
177
+ samples.put(sample)
178
+ }
179
+ out.put("samples", samples).put("ok", true)
180
+ }
181
+ } catch (e: Exception) {
182
+ out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
183
+ }
184
+ return out.toString()
185
+ }
186
+
187
+ /**
188
+ * Every sample timestamp, per track, read from the container.
189
+ *
190
+ * `MediaMetadataRetriever` reports a frame *count* and a whole-file
191
+ * duration, neither of which can distinguish a held tail frame from a
192
+ * correctly-timed one, or a gap from a slow rate. The muxed sample
193
+ * timestamps can: they are exactly what was written.
194
+ *
195
+ * As with everything else here, it reports and concludes nothing. Whether an
196
+ * interval is a gap, a duplicate or expected slow-motion resampling is a
197
+ * judgement the TypeScript oracle makes against a rate it declared first.
198
+ */
199
+ fun tracks(context: Context, uri: String): String {
200
+ val out = JSONObject()
201
+ val extractor = android.media.MediaExtractor()
202
+ try {
203
+ if (uri.startsWith("/")) {
204
+ extractor.setDataSource(uri)
205
+ } else {
206
+ extractor.setDataSource(context, Uri.parse(uri), null)
207
+ }
208
+ val tracks = JSONArray()
209
+ val count = extractor.trackCount
210
+ val mimes = arrayOfNulls<String>(count)
211
+ val counts = IntArray(count)
212
+ val firsts = LongArray(count) { -1L }
213
+ val lasts = LongArray(count) { -1L }
214
+ val videoTimes = HashMap<Int, JSONArray>()
215
+ for (i in 0 until count) {
216
+ val format = extractor.getTrackFormat(i)
217
+ mimes[i] = format.getString(android.media.MediaFormat.KEY_MIME) ?: ""
218
+ // Every track selected, one pass, sorted by the reader itself.
219
+ // Selecting and seeking per track made an empty audio track and
220
+ // an unseekable one indistinguishable; reading the interleaved
221
+ // stream and asking which track each sample belongs to cannot
222
+ // confuse the two.
223
+ extractor.selectTrack(i)
224
+ if (mimes[i]!!.startsWith("video/")) videoTimes[i] = JSONArray()
225
+ }
226
+ extractor.seekTo(0, android.media.MediaExtractor.SEEK_TO_PREVIOUS_SYNC)
227
+ while (true) {
228
+ val track = extractor.sampleTrackIndex
229
+ if (track < 0) break
230
+ val t = extractor.sampleTime
231
+ if (counts[track] == 0) firsts[track] = t
232
+ lasts[track] = t
233
+ videoTimes[track]?.put(t)
234
+ counts[track] += 1
235
+ if (!extractor.advance()) break
236
+ }
237
+ for (i in 0 until count) {
238
+ val format = extractor.getTrackFormat(i)
239
+ val track = JSONObject()
240
+ .put("index", i)
241
+ .put("mime", mimes[i])
242
+ .put(
243
+ "formatDurationUs",
244
+ if (format.containsKey(android.media.MediaFormat.KEY_DURATION)) {
245
+ format.getLong(android.media.MediaFormat.KEY_DURATION)
246
+ } else {
247
+ -1L
248
+ },
249
+ )
250
+ .put("sampleCount", counts[i])
251
+ .put("firstPtsUs", firsts[i])
252
+ .put("lastPtsUs", lasts[i])
253
+ videoTimes[i]?.let { track.put("timestampsUs", it) }
254
+ tracks.put(track)
255
+ }
256
+ out.put("tracks", tracks).put("ok", true)
257
+ } catch (e: Exception) {
258
+ out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
259
+ } finally {
260
+ runCatching { extractor.release() }
261
+ }
262
+ return out.toString()
263
+ }
264
+
265
+ /**
266
+ * A checksum per compressed video sample, without decoding.
267
+ *
268
+ * Transmuxing must move the encoded bytes and change only timestamps and
269
+ * container metadata. Comparing decoded pixels cannot show that: a re-encode
270
+ * that happens to look identical would pass, and this gate exists to tell
271
+ * those apart. So the payload itself is checksummed.
272
+ */
273
+ fun sampleHashes(context: Context, uri: String): String {
274
+ val out = JSONObject()
275
+ val extractor = android.media.MediaExtractor()
276
+ try {
277
+ if (uri.startsWith("/")) {
278
+ extractor.setDataSource(uri)
279
+ } else {
280
+ extractor.setDataSource(context, Uri.parse(uri), null)
281
+ }
282
+ var video = -1
283
+ for (i in 0 until extractor.trackCount) {
284
+ val mime = extractor.getTrackFormat(i).getString(android.media.MediaFormat.KEY_MIME) ?: ""
285
+ if (mime.startsWith("video/")) { video = i; break }
286
+ }
287
+ /*
288
+ An audio-only file is a legitimate input here.
289
+
290
+ The audio timeline intermediate has no video track, and refusing it
291
+ made the payload comparison unreadable — it reported "no audio
292
+ hashes" for the one file that was nothing but audio.
293
+ */
294
+ /*
295
+ One interleaved pass over every track.
296
+
297
+ Selecting and seeking per track reads nothing when a track's first
298
+ sample has a negative timestamp, which AAC's encoder delay routinely
299
+ produces -- the audio hashes came back empty for exactly the files
300
+ that had audio. Reading the stream as the extractor orders it and
301
+ asking which track each sample belongs to cannot make that mistake.
302
+ */
303
+ for (i in 0 until extractor.trackCount) extractor.selectTrack(i)
304
+ extractor.seekTo(0, android.media.MediaExtractor.SEEK_TO_PREVIOUS_SYNC)
305
+ val buffer = java.nio.ByteBuffer.allocate(4 * 1024 * 1024)
306
+
307
+ val hashes = JSONArray()
308
+ val sizes = JSONArray()
309
+ val times = JSONArray()
310
+ val audioHashes = JSONArray()
311
+ val audioSizes = JSONArray()
312
+ val audioTimes = JSONArray()
313
+ val whole = java.util.zip.CRC32()
314
+ val beatEpoch = ExportHeartbeat.currentEpoch()
315
+ var beatCounter = 0
316
+ while (true) {
317
+ val track = extractor.sampleTrackIndex
318
+ if (track < 0) break
319
+ val t = extractor.sampleTime
320
+ buffer.clear()
321
+ val read = extractor.readSampleData(buffer, 0)
322
+ // Provenance hashing reads the whole file, and measured at
323
+ // 0.1x it was the longest unbeaten stretch of an entire export
324
+ // — 5.5 s of silence in a pipeline that was working perfectly.
325
+ // Conformance work beats too, so a gate run does not look
326
+ // stalled — but production does not run it at all, so the
327
+ // threshold is not measured against it.
328
+ beatCounter += 1
329
+ if (beatCounter % 64 == 0) ExportHeartbeat.beat("muxBatch", beatEpoch)
330
+ if (read <= 0) break
331
+ val bytes = ByteArray(read)
332
+ buffer.position(0)
333
+ buffer.get(bytes, 0, read)
334
+ val crc = java.util.zip.CRC32()
335
+ crc.update(bytes, 0, read)
336
+ if (track == video) {
337
+ whole.update(bytes, 0, read)
338
+ hashes.put(crc.value)
339
+ sizes.put(read)
340
+ times.put(t)
341
+ } else {
342
+ audioHashes.put(crc.value)
343
+ audioSizes.put(read)
344
+ audioTimes.put(t)
345
+ }
346
+ if (!extractor.advance()) break
347
+ }
348
+ out.put("sampleHashes", hashes)
349
+ .put("sampleSizes", sizes)
350
+ .put("timestampsUs", times)
351
+ .put("audioHashes", audioHashes)
352
+ .put("audioSizes", audioSizes)
353
+ .put("audioTimestampsUs", audioTimes)
354
+ .put("payloadCrc", whole.value)
355
+ .put("ok", true)
356
+ } catch (e: Exception) {
357
+ out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
358
+ } finally {
359
+ runCatching { extractor.release() }
360
+ }
361
+ return out.toString()
362
+ }
363
+
364
+ /**
365
+ * Geometry and colour, separated.
366
+ *
367
+ * A maximum channel delta is close to useless around a transformed edge: one
368
+ * row of pixels landing on the far side of a boundary between a bright source
369
+ * and a black background produces a delta of 255 while the picture is right.
370
+ * So the frame is split into a **mask** — where the transformed source landed
371
+ * — and the colour inside it, and each is measured on its own terms.
372
+ *
373
+ * Reports numbers only. Which of them constitute a pass is the caller's
374
+ * predeclared business.
375
+ */
376
+ fun geometry(
377
+ context: Context,
378
+ leftUri: String,
379
+ leftUs: Long,
380
+ rightUri: String,
381
+ rightUs: Long,
382
+ maskThreshold: Int,
383
+ ): String {
384
+ val out = JSONObject()
385
+ try {
386
+ Reader(context, leftUri).use { l ->
387
+ Reader(context, rightUri).use { r ->
388
+ val a = l.frameAt(leftUs)
389
+ val b = r.frameAt(rightUs)
390
+ if (a == null || b == null) return out.put("ok", false).put("error", "no frame").toString()
391
+ if (a.width != b.width || a.height != b.height) {
392
+ return out.put("ok", false)
393
+ .put("error", "size ${a.width}x${a.height} vs ${b.width}x${b.height}")
394
+ .toString()
395
+ }
396
+ val w = a.width
397
+ val h = a.height
398
+ val pa = IntArray(w * h)
399
+ val pb = IntArray(w * h)
400
+ a.getPixels(pa, 0, w, 0, 0, w, h)
401
+ b.getPixels(pb, 0, w, 0, 0, w, h)
402
+ a.recycle()
403
+ b.recycle()
404
+
405
+ val ma = BooleanArray(w * h)
406
+ val mb = BooleanArray(w * h)
407
+ var aBox = intArrayOf(w, h, -1, -1)
408
+ var bBox = intArrayOf(w, h, -1, -1)
409
+ var areaA = 0L
410
+ var areaB = 0L
411
+ for (y in 0 until h) {
412
+ for (x in 0 until w) {
413
+ val i = y * w + x
414
+ if (maskValue(pa[i]) > maskThreshold) {
415
+ ma[i] = true; areaA++
416
+ if (x < aBox[0]) aBox[0] = x
417
+ if (y < aBox[1]) aBox[1] = y
418
+ if (x > aBox[2]) aBox[2] = x
419
+ if (y > aBox[3]) aBox[3] = y
420
+ }
421
+ if (maskValue(pb[i]) > maskThreshold) {
422
+ mb[i] = true; areaB++
423
+ if (x < bBox[0]) bBox[0] = x
424
+ if (y < bBox[1]) bBox[1] = y
425
+ if (x > bBox[2]) bBox[2] = x
426
+ if (y > bBox[3]) bBox[3] = y
427
+ }
428
+ }
429
+ }
430
+ var inter = 0L
431
+ var union = 0L
432
+ for (i in 0 until w * h) {
433
+ if (ma[i] && mb[i]) inter++
434
+ if (ma[i] || mb[i]) union++
435
+ }
436
+ /*
437
+ An empty union is not a perfect match.
438
+
439
+ Two frames that both drew nothing agree completely and mean
440
+ nothing, and IoU over an empty set is whatever the guard
441
+ says it is. Four transform cases once "passed" at 1000
442
+ permille while rendering black, which is the shape of
443
+ failure this refuses to report as success.
444
+ */
445
+ if (union == 0L) {
446
+ return out.put("ok", false)
447
+ .put("error", "empty union: neither frame drew a footprint")
448
+ .put("maskAreaLeft", areaA).put("maskAreaRight", areaB)
449
+ .toString()
450
+ }
451
+
452
+ /*
453
+ Interior only: the intersection, eroded by one pixel.
454
+
455
+ An edge pixel is a blend of source and background whose
456
+ value depends on exactly where the boundary fell, so it
457
+ reports colour error for what is really sub-pixel geometry.
458
+ Eroding separates the two questions instead of letting one
459
+ answer the other.
460
+ */
461
+ var interiorPixels = 0L
462
+ var sumErr = 0L
463
+ var maxErr = 0
464
+ var differing = 0L
465
+ for (y in 1 until h - 1) {
466
+ for (x in 1 until w - 1) {
467
+ val i = y * w + x
468
+ if (!(ma[i] && mb[i])) continue
469
+ if (!(ma[i - 1] && ma[i + 1] && ma[i - w] && ma[i + w])) continue
470
+ if (!(mb[i - 1] && mb[i + 1] && mb[i - w] && mb[i + w])) continue
471
+ val p = pa[i]
472
+ val q = pb[i]
473
+ val dr = abs(((p shr 16) and 0xff) - ((q shr 16) and 0xff))
474
+ val dg = abs(((p shr 8) and 0xff) - ((q shr 8) and 0xff))
475
+ val db = abs((p and 0xff) - (q and 0xff))
476
+ val worst = maxOf(dr, dg, db)
477
+ if (worst > maxErr) maxErr = worst
478
+ sumErr += (dr + dg + db).toLong()
479
+ if (worst > 8) differing++
480
+ interiorPixels++
481
+ }
482
+ }
483
+
484
+ // Edge displacement, as how far the footprints' bounds moved.
485
+ var edge = 0
486
+ if (aBox[2] >= 0 && bBox[2] >= 0) {
487
+ for (k in 0 until 4) edge = maxOf(edge, abs(aBox[k] - bBox[k]))
488
+ }
489
+
490
+ out.put("ok", true)
491
+ .put("width", w).put("height", h)
492
+ .put("maskAreaLeft", areaA).put("maskAreaRight", areaB)
493
+ .put("intersection", inter).put("union", union)
494
+ .put("iouPermille", if (union == 0L) 1000 else ((inter * 1000) / union).toInt())
495
+ .put("symmetricDifference", union - inter)
496
+ .put("bboxLeft", JSONArray().put(aBox[0]).put(aBox[1]).put(aBox[2]).put(aBox[3]))
497
+ .put("bboxRight", JSONArray().put(bBox[0]).put(bBox[1]).put(bBox[2]).put(bBox[3]))
498
+ .put("edgeDisplacementPx", edge)
499
+ .put("interiorPixels", interiorPixels)
500
+ .put("interiorMeanError", if (interiorPixels == 0L) 0.0 else sumErr.toDouble() / (interiorPixels * 3))
501
+ .put("interiorMaxError", maxErr)
502
+ .put(
503
+ "interiorDifferingPermille",
504
+ if (interiorPixels == 0L) 0 else ((differing * 1000) / interiorPixels).toInt(),
505
+ )
506
+ }
507
+ }
508
+ } catch (e: Exception) {
509
+ out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
510
+ }
511
+ return out.toString()
512
+ }
513
+
514
+ /** One decoded frame as base64 PNG, in display space. */
515
+ fun framePng(context: Context, uri: String, timeUs: Long): String {
516
+ val out = JSONObject()
517
+ try {
518
+ Reader(context, uri).use { reader ->
519
+ val bitmap = reader.frameAt(timeUs)
520
+ ?: return out.put("ok", false).put("error", "no frame").toString()
521
+ val stream = java.io.ByteArrayOutputStream()
522
+ bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
523
+ out.put("ok", true)
524
+ .put("width", bitmap.width)
525
+ .put("height", bitmap.height)
526
+ .put("png", android.util.Base64.encodeToString(stream.toByteArray(), android.util.Base64.NO_WRAP))
527
+ bitmap.recycle()
528
+ }
529
+ } catch (e: Exception) {
530
+ out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
531
+ }
532
+ return out.toString()
533
+ }
534
+
535
+ /**
536
+ * Mask test: the brightest channel, not luminance.
537
+ *
538
+ * Luminance weights blue at 11%, so a blue-only source at value 93 scores 13
539
+ * — indistinguishable from black at any usable threshold, and the mask
540
+ * collapsed to the burned-in text instead of the source's footprint. The
541
+ * question here is "is there source here at all", and the strongest channel
542
+ * answers it without caring which channel carries the signal.
543
+ */
544
+ /**
545
+ * Mean and extremes over caller-chosen rectangles.
546
+ *
547
+ * The fixed centre region [sample] uses is wrong for a transformed clip: it
548
+ * straddles the footprint edge, so it averages source pixels with cleared
549
+ * ones and the recovered source frame reads low. The caller knows where the
550
+ * footprint is — it has the corner oracle — so it passes the rectangles and
551
+ * this reports what is in them.
552
+ *
553
+ * `max - min` per channel is returned as well as the mean, because a region
554
+ * that is supposed to be uniform and is not is a different failure from one
555
+ * whose average is off.
556
+ */
557
+ fun probe(context: Context, uri: String, probesJson: String): String {
558
+ val out = JSONObject()
559
+ try {
560
+ val probes = JSONArray(probesJson)
561
+ Reader(context, uri).use { reader ->
562
+ val results = JSONArray()
563
+ var lastUs = Long.MIN_VALUE
564
+ var bitmap: Bitmap? = null
565
+ for (i in 0 until probes.length()) {
566
+ val p = probes.getJSONObject(i)
567
+ val us = p.getLong("timeUs")
568
+ if (us != lastUs) {
569
+ bitmap?.recycle()
570
+ bitmap = reader.frameAt(us)
571
+ lastUs = us
572
+ }
573
+ val bm = bitmap
574
+ val entry = JSONObject().put("timeUs", us)
575
+ .put("x", p.getInt("x")).put("y", p.getInt("y"))
576
+ .put("w", p.getInt("w")).put("h", p.getInt("h"))
577
+ if (bm == null) {
578
+ results.put(entry.put("error", "no frame")); continue
579
+ }
580
+ val x0 = p.getInt("x").coerceIn(0, bm.width - 1)
581
+ val y0 = p.getInt("y").coerceIn(0, bm.height - 1)
582
+ val w = p.getInt("w").coerceIn(1, bm.width - x0)
583
+ val h = p.getInt("h").coerceIn(1, bm.height - y0)
584
+ val row = IntArray(w)
585
+ var r = 0L; var g = 0L; var b = 0L
586
+ var rMin = 255; var gMin = 255; var bMin = 255
587
+ var rMax = 0; var gMax = 0; var bMax = 0
588
+ for (y in y0 until y0 + h) {
589
+ bm.getPixels(row, 0, w, x0, y, w, 1)
590
+ for (px in row) {
591
+ val pr = (px shr 16) and 0xff
592
+ val pg = (px shr 8) and 0xff
593
+ val pb = px and 0xff
594
+ r += pr; g += pg; b += pb
595
+ if (pr < rMin) rMin = pr
596
+ if (pg < gMin) gMin = pg
597
+ if (pb < bMin) bMin = pb
598
+ if (pr > rMax) rMax = pr
599
+ if (pg > gMax) gMax = pg
600
+ if (pb > bMax) bMax = pb
601
+ }
602
+ }
603
+ val n = w.toLong() * h.toLong()
604
+ results.put(
605
+ entry.put("r", r.toDouble() / n).put("g", g.toDouble() / n).put("b", b.toDouble() / n)
606
+ .put("rMin", rMin).put("gMin", gMin).put("bMin", bMin)
607
+ .put("rMax", rMax).put("gMax", gMax).put("bMax", bMax)
608
+ .put("pixels", n),
609
+ )
610
+ }
611
+ bitmap?.recycle()
612
+ out.put("probes", results).put("ok", true)
613
+ }
614
+ } catch (e: Exception) {
615
+ out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
616
+ }
617
+ return out.toString()
618
+ }
619
+
620
+ private fun maskValue(p: Int): Int {
621
+ val r = (p shr 16) and 0xff
622
+ val g = (p shr 8) and 0xff
623
+ val b = p and 0xff
624
+ return maxOf(r, maxOf(g, b))
625
+ }
626
+
627
+ private class Stats(val maxDelta: Int, val meanDelta: Double, val differingPermille: Int)
628
+
629
+ private fun diff(a: Bitmap, b: Bitmap): Stats {
630
+ val w = a.width
631
+ val h = a.height
632
+ val rowA = IntArray(w)
633
+ val rowB = IntArray(w)
634
+ var max = 0
635
+ var sum = 0L
636
+ var differing = 0L
637
+ var total = 0L
638
+ for (y in 0 until h) {
639
+ a.getPixels(rowA, 0, w, 0, y, w, 1)
640
+ b.getPixels(rowB, 0, w, 0, y, w, 1)
641
+ for (x in 0 until w) {
642
+ val p = rowA[x]
643
+ val q = rowB[x]
644
+ val dr = abs(((p shr 16) and 0xff) - ((q shr 16) and 0xff))
645
+ val dg = abs(((p shr 8) and 0xff) - ((q shr 8) and 0xff))
646
+ val db = abs((p and 0xff) - (q and 0xff))
647
+ if (dr > max) max = dr
648
+ if (dg > max) max = dg
649
+ if (db > max) max = db
650
+ sum += dr + dg + db
651
+ if (dr != 0 || dg != 0 || db != 0) differing += 1
652
+ total += 3
653
+ }
654
+ }
655
+ val pixels = w.toLong() * h.toLong()
656
+ return Stats(
657
+ maxDelta = max,
658
+ meanDelta = if (total == 0L) 0.0 else sum.toDouble() / total.toDouble(),
659
+ differingPermille = if (pixels == 0L) 0 else ((differing * 1000) / pixels).toInt(),
660
+ )
661
+ }
662
+ }