@pictor-sdk/video-android 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +40 -0
- package/android/CMakeLists.txt +30 -0
- package/android/build.gradle +119 -0
- package/android/src/main/AndroidManifest.xml +1 -0
- package/android/src/main/cpp/pictor_overlaystage.cpp +372 -0
- package/android/src/main/cpp/pictor_publish.cpp +87 -0
- package/android/src/main/cpp/pictor_videosurface.cpp +735 -0
- package/android/src/main/java/dev/pictor/video/AacPriming.kt +261 -0
- package/android/src/main/java/dev/pictor/video/AudioMixer.kt +1112 -0
- package/android/src/main/java/dev/pictor/video/AudioProbe.kt +162 -0
- package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt +337 -0
- package/android/src/main/java/dev/pictor/video/CodecLedger.kt +126 -0
- package/android/src/main/java/dev/pictor/video/CompositionSpike.kt +978 -0
- package/android/src/main/java/dev/pictor/video/Diagnostics.kt +54 -0
- package/android/src/main/java/dev/pictor/video/EncoderGate.kt +476 -0
- package/android/src/main/java/dev/pictor/video/ExportGate.kt +1974 -0
- package/android/src/main/java/dev/pictor/video/ExportHeartbeat.kt +72 -0
- package/android/src/main/java/dev/pictor/video/ExportProgress.kt +103 -0
- package/android/src/main/java/dev/pictor/video/FrameClock.kt +591 -0
- package/android/src/main/java/dev/pictor/video/FramePending.kt +181 -0
- package/android/src/main/java/dev/pictor/video/GateAudio.kt +456 -0
- package/android/src/main/java/dev/pictor/video/GateDestination.kt +445 -0
- package/android/src/main/java/dev/pictor/video/GateFrames.kt +662 -0
- package/android/src/main/java/dev/pictor/video/GateMux.kt +185 -0
- package/android/src/main/java/dev/pictor/video/ManagedSources.kt +333 -0
- package/android/src/main/java/dev/pictor/video/Media3Player.kt +934 -0
- package/android/src/main/java/dev/pictor/video/MediaStoreSave.kt +419 -0
- package/android/src/main/java/dev/pictor/video/OverlayResources.kt +169 -0
- package/android/src/main/java/dev/pictor/video/PictorVideoModule.kt +1612 -0
- package/android/src/main/java/dev/pictor/video/PictorVideoPackage.kt +26 -0
- package/android/src/main/java/dev/pictor/video/PlaybackAnchor.kt +161 -0
- package/android/src/main/java/dev/pictor/video/PreviewAudioPlayer.kt +291 -0
- package/android/src/main/java/dev/pictor/video/PreviewConfig.kt +96 -0
- package/android/src/main/java/dev/pictor/video/PreviewMix.kt +396 -0
- package/android/src/main/java/dev/pictor/video/PreviewSync.kt +374 -0
- package/android/src/main/java/dev/pictor/video/PreviewTrace.kt +478 -0
- package/android/src/main/java/dev/pictor/video/PrimingTrim.kt +337 -0
- package/android/src/main/java/dev/pictor/video/RepeatingEncoderFactory.kt +97 -0
- package/android/src/main/java/dev/pictor/video/SceneProgram.kt +169 -0
- package/android/src/main/java/dev/pictor/video/SceneProgramNative.kt +299 -0
- package/android/src/main/java/dev/pictor/video/SkiaPassThroughEffect.kt +695 -0
- package/android/src/main/java/dev/pictor/video/SurfaceAttachment.kt +110 -0
- package/android/src/main/java/dev/pictor/video/ThumbnailDecoder.kt +171 -0
- package/android/src/main/java/dev/pictor/video/VideoSurface.kt +394 -0
- package/app.plugin.js +43 -0
- package/dist/index.d.mts +3622 -0
- package/dist/index.d.ts +3622 -0
- package/dist/index.js +3737 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +3625 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +82 -0
- package/plugin/gradleProperties.d.ts +19 -0
- package/plugin/gradleProperties.js +64 -0
- package/react-native.config.js +13 -0
- package/src/NativePictorVideo.ts +574 -0
- package/src/audioProbe.ts +72 -0
- package/src/compositionSpike.ts +305 -0
- package/src/convergence.ts +112 -0
- package/src/encoderGate.ts +66 -0
- package/src/exportDestination.ts +149 -0
- package/src/exportJob.ts +232 -0
- package/src/exportProgress.ts +391 -0
- package/src/exportProgressPump.ts +190 -0
- package/src/exporter.ts +655 -0
- package/src/index.ts +276 -0
- package/src/managedSources.ts +252 -0
- package/src/media3.ts +412 -0
- package/src/mediaStore.ts +177 -0
- package/src/memoryGuard.ts +118 -0
- package/src/mixSources.ts +82 -0
- package/src/overlayStaging.ts +216 -0
- package/src/playbackLifecycle.ts +137 -0
- package/src/player.ts +39 -0
- package/src/playlistSession.ts +125 -0
- package/src/previewAudioDiagnostics.ts +67 -0
- package/src/previewAudioSession.ts +168 -0
- package/src/previewCadence.ts +950 -0
- package/src/previewMix.ts +212 -0
- package/src/previewTrace.ts +348 -0
- package/src/spikeExcursions.ts +405 -0
- package/src/submissionScoring.ts +680 -0
- package/src/surfaceOwner.ts +264 -0
- package/src/surfaceRecovery.ts +121 -0
- package/src/syncTrace.ts +189 -0
- package/src/thumbnailProvider.ts +163 -0
- package/src/tracePhases.ts +217 -0
- package/src/useCompositionPlayback.ts +327 -0
- package/src/useMedia3Playback.ts +492 -0
- package/src/usePreviewAudio.ts +305 -0
- package/src/useVideoSurface.ts +282 -0
|
@@ -0,0 +1,662 @@
|
|
|
1
|
+
package dev.pictor.video
|
|
2
|
+
|
|
3
|
+
import android.content.Context
|
|
4
|
+
import android.graphics.Bitmap
|
|
5
|
+
import android.media.MediaMetadataRetriever
|
|
6
|
+
import android.net.Uri
|
|
7
|
+
import org.json.JSONArray
|
|
8
|
+
import org.json.JSONObject
|
|
9
|
+
import kotlin.math.abs
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Decoded **display-space** frames, for the gate's pixel floor.
|
|
13
|
+
*
|
|
14
|
+
* `MediaMetadataRetriever` returns frames with the container's rotation already
|
|
15
|
+
* applied, which is the only space in which "does this look the same" is a
|
|
16
|
+
* meaningful question. Comparing stored dimensions instead is how a 1080×1920
|
|
17
|
+
* file carrying a rotation flag and a 1920×1080 file with the orientation baked
|
|
18
|
+
* in look like a difference when they display identically — the exact ambiguity
|
|
19
|
+
* that stalled the first run of this gate.
|
|
20
|
+
*
|
|
21
|
+
* Nothing here encodes or exports. It reads files back and reports numbers.
|
|
22
|
+
*/
|
|
23
|
+
internal object GateFrames {
|
|
24
|
+
|
|
25
|
+
/** What one file looks like, in display space. */
|
|
26
|
+
private class Reader(context: Context, uri: String) : AutoCloseable {
|
|
27
|
+
val mmr = MediaMetadataRetriever()
|
|
28
|
+
|
|
29
|
+
init {
|
|
30
|
+
if (uri.startsWith("/")) {
|
|
31
|
+
mmr.setDataSource(uri)
|
|
32
|
+
} else {
|
|
33
|
+
mmr.setDataSource(context, Uri.parse(uri))
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
fun meta(key: Int): String = mmr.extractMetadata(key) ?: ""
|
|
38
|
+
|
|
39
|
+
fun frameAt(us: Long): Bitmap? =
|
|
40
|
+
mmr.getFrameAtTime(us, MediaMetadataRetriever.OPTION_CLOSEST)
|
|
41
|
+
|
|
42
|
+
override fun close() {
|
|
43
|
+
runCatching { mmr.release() }
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Compare two files at `timestampsUs`, in display space.
|
|
49
|
+
*
|
|
50
|
+
* Returns per-timestamp statistics rather than a verdict: the threshold is
|
|
51
|
+
* the caller's to freeze, and freezing it here would be choosing it after
|
|
52
|
+
* seeing the numbers.
|
|
53
|
+
*/
|
|
54
|
+
fun compare(
|
|
55
|
+
context: Context,
|
|
56
|
+
leftUri: String,
|
|
57
|
+
rightUri: String,
|
|
58
|
+
timestampsUs: LongArray,
|
|
59
|
+
): String {
|
|
60
|
+
val out = JSONObject()
|
|
61
|
+
try {
|
|
62
|
+
Reader(context, leftUri).use { left ->
|
|
63
|
+
Reader(context, rightUri).use { right ->
|
|
64
|
+
out.put("leftDisplay", describe(left))
|
|
65
|
+
out.put("rightDisplay", describe(right))
|
|
66
|
+
val samples = JSONArray()
|
|
67
|
+
var worstMax = 0
|
|
68
|
+
var worstMeanTimesThousand = 0L
|
|
69
|
+
for (us in timestampsUs) {
|
|
70
|
+
val a = left.frameAt(us)
|
|
71
|
+
val b = right.frameAt(us)
|
|
72
|
+
val sample = JSONObject().put("timeUs", us)
|
|
73
|
+
if (a == null || b == null) {
|
|
74
|
+
sample.put("error", "no frame")
|
|
75
|
+
} else if (a.width != b.width || a.height != b.height) {
|
|
76
|
+
// A real difference, and not one a tolerance can
|
|
77
|
+
// absorb: the two do not display at the same size.
|
|
78
|
+
sample.put("error", "display size ${a.width}x${a.height} vs ${b.width}x${b.height}")
|
|
79
|
+
} else {
|
|
80
|
+
val stats = diff(a, b)
|
|
81
|
+
sample.put("width", a.width).put("height", a.height)
|
|
82
|
+
sample.put("maxDelta", stats.maxDelta)
|
|
83
|
+
sample.put("meanDelta", stats.meanDelta)
|
|
84
|
+
sample.put("differingPermille", stats.differingPermille)
|
|
85
|
+
if (stats.maxDelta > worstMax) worstMax = stats.maxDelta
|
|
86
|
+
val meanK = (stats.meanDelta * 1000).toLong()
|
|
87
|
+
if (meanK > worstMeanTimesThousand) worstMeanTimesThousand = meanK
|
|
88
|
+
}
|
|
89
|
+
a?.recycle()
|
|
90
|
+
b?.recycle()
|
|
91
|
+
samples.put(sample)
|
|
92
|
+
}
|
|
93
|
+
out.put("samples", samples)
|
|
94
|
+
out.put("worstMaxDelta", worstMax)
|
|
95
|
+
out.put("worstMeanDelta", worstMeanTimesThousand / 1000.0)
|
|
96
|
+
out.put("ok", true)
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
} catch (e: Exception) {
|
|
100
|
+
out.put("ok", false).put("error", "${e.javaClass.simpleName}: ${e.message}")
|
|
101
|
+
}
|
|
102
|
+
return out.toString()
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private fun describe(reader: Reader): JSONObject {
|
|
106
|
+
// Stored dimensions *and* the rotation, so display space can be derived
|
|
107
|
+
// rather than guessed — and a first frame, which is display space by
|
|
108
|
+
// construction.
|
|
109
|
+
val storedW = reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)
|
|
110
|
+
val storedH = reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)
|
|
111
|
+
val rotation = reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)
|
|
112
|
+
val first = reader.frameAt(0)
|
|
113
|
+
val json = JSONObject()
|
|
114
|
+
.put("storedWidth", storedW)
|
|
115
|
+
.put("storedHeight", storedH)
|
|
116
|
+
.put("rotationDegrees", rotation)
|
|
117
|
+
.put("durationMs", reader.meta(MediaMetadataRetriever.METADATA_KEY_DURATION))
|
|
118
|
+
.put("frameCount", reader.meta(MediaMetadataRetriever.METADATA_KEY_VIDEO_FRAME_COUNT))
|
|
119
|
+
.put("colorStandard", reader.meta(MediaMetadataRetriever.METADATA_KEY_COLOR_STANDARD))
|
|
120
|
+
.put("colorRange", reader.meta(MediaMetadataRetriever.METADATA_KEY_COLOR_RANGE))
|
|
121
|
+
.put("colorTransfer", reader.meta(MediaMetadataRetriever.METADATA_KEY_COLOR_TRANSFER))
|
|
122
|
+
.put("hasAudio", reader.meta(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO))
|
|
123
|
+
.put("displayWidth", first?.width ?: 0)
|
|
124
|
+
.put("displayHeight", first?.height ?: 0)
|
|
125
|
+
first?.recycle()
|
|
126
|
+
return json
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Mean colour of a central region, per timestamp, in display space.
|
|
131
|
+
*
|
|
132
|
+
* Deliberately dumb: it reports numbers and draws no conclusion. Which
|
|
133
|
+
* source a colour implies, and which frame of it, is arithmetic the
|
|
134
|
+
* TypeScript oracle owns — native computing that would be exactly the
|
|
135
|
+
* timeline knowledge this architecture keeps out of it.
|
|
136
|
+
*
|
|
137
|
+
* The region avoids the burned-in label, so text pixels cannot pull the
|
|
138
|
+
* mean around.
|
|
139
|
+
*/
|
|
140
|
+
fun sample(context: Context, uri: String, timestampsUs: LongArray): String {
|
|
141
|
+
val out = JSONObject()
|
|
142
|
+
try {
|
|
143
|
+
Reader(context, uri).use { reader ->
|
|
144
|
+
out.put("display", describe(reader))
|
|
145
|
+
val samples = JSONArray()
|
|
146
|
+
for (us in timestampsUs) {
|
|
147
|
+
val bitmap = reader.frameAt(us)
|
|
148
|
+
val sample = JSONObject().put("timeUs", us)
|
|
149
|
+
if (bitmap == null) {
|
|
150
|
+
sample.put("error", "no frame")
|
|
151
|
+
} else {
|
|
152
|
+
val w = bitmap.width
|
|
153
|
+
val h = bitmap.height
|
|
154
|
+
// Lower-middle: below the label, inside the field.
|
|
155
|
+
val x0 = w / 4
|
|
156
|
+
val y0 = h / 2
|
|
157
|
+
val cw = w / 2
|
|
158
|
+
val ch = h / 3
|
|
159
|
+
val row = IntArray(cw)
|
|
160
|
+
var r = 0L
|
|
161
|
+
var g = 0L
|
|
162
|
+
var b = 0L
|
|
163
|
+
for (y in y0 until y0 + ch) {
|
|
164
|
+
bitmap.getPixels(row, 0, cw, x0, y, cw, 1)
|
|
165
|
+
for (p in row) {
|
|
166
|
+
r += (p shr 16) and 0xff
|
|
167
|
+
g += (p shr 8) and 0xff
|
|
168
|
+
b += p and 0xff
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
val n = cw.toLong() * ch.toLong()
|
|
172
|
+
sample.put("r", r.toDouble() / n)
|
|
173
|
+
sample.put("g", g.toDouble() / n)
|
|
174
|
+
sample.put("b", b.toDouble() / n)
|
|
175
|
+
bitmap.recycle()
|
|
176
|
+
}
|
|
177
|
+
samples.put(sample)
|
|
178
|
+
}
|
|
179
|
+
out.put("samples", samples).put("ok", true)
|
|
180
|
+
}
|
|
181
|
+
} catch (e: Exception) {
|
|
182
|
+
out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
|
|
183
|
+
}
|
|
184
|
+
return out.toString()
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Every sample timestamp, per track, read from the container.
|
|
189
|
+
*
|
|
190
|
+
* `MediaMetadataRetriever` reports a frame *count* and a whole-file
|
|
191
|
+
* duration, neither of which can distinguish a held tail frame from a
|
|
192
|
+
* correctly-timed one, or a gap from a slow rate. The muxed sample
|
|
193
|
+
* timestamps can: they are exactly what was written.
|
|
194
|
+
*
|
|
195
|
+
* As with everything else here, it reports and concludes nothing. Whether an
|
|
196
|
+
* interval is a gap, a duplicate or expected slow-motion resampling is a
|
|
197
|
+
* judgement the TypeScript oracle makes against a rate it declared first.
|
|
198
|
+
*/
|
|
199
|
+
fun tracks(context: Context, uri: String): String {
|
|
200
|
+
val out = JSONObject()
|
|
201
|
+
val extractor = android.media.MediaExtractor()
|
|
202
|
+
try {
|
|
203
|
+
if (uri.startsWith("/")) {
|
|
204
|
+
extractor.setDataSource(uri)
|
|
205
|
+
} else {
|
|
206
|
+
extractor.setDataSource(context, Uri.parse(uri), null)
|
|
207
|
+
}
|
|
208
|
+
val tracks = JSONArray()
|
|
209
|
+
val count = extractor.trackCount
|
|
210
|
+
val mimes = arrayOfNulls<String>(count)
|
|
211
|
+
val counts = IntArray(count)
|
|
212
|
+
val firsts = LongArray(count) { -1L }
|
|
213
|
+
val lasts = LongArray(count) { -1L }
|
|
214
|
+
val videoTimes = HashMap<Int, JSONArray>()
|
|
215
|
+
for (i in 0 until count) {
|
|
216
|
+
val format = extractor.getTrackFormat(i)
|
|
217
|
+
mimes[i] = format.getString(android.media.MediaFormat.KEY_MIME) ?: ""
|
|
218
|
+
// Every track selected, one pass, sorted by the reader itself.
|
|
219
|
+
// Selecting and seeking per track made an empty audio track and
|
|
220
|
+
// an unseekable one indistinguishable; reading the interleaved
|
|
221
|
+
// stream and asking which track each sample belongs to cannot
|
|
222
|
+
// confuse the two.
|
|
223
|
+
extractor.selectTrack(i)
|
|
224
|
+
if (mimes[i]!!.startsWith("video/")) videoTimes[i] = JSONArray()
|
|
225
|
+
}
|
|
226
|
+
extractor.seekTo(0, android.media.MediaExtractor.SEEK_TO_PREVIOUS_SYNC)
|
|
227
|
+
while (true) {
|
|
228
|
+
val track = extractor.sampleTrackIndex
|
|
229
|
+
if (track < 0) break
|
|
230
|
+
val t = extractor.sampleTime
|
|
231
|
+
if (counts[track] == 0) firsts[track] = t
|
|
232
|
+
lasts[track] = t
|
|
233
|
+
videoTimes[track]?.put(t)
|
|
234
|
+
counts[track] += 1
|
|
235
|
+
if (!extractor.advance()) break
|
|
236
|
+
}
|
|
237
|
+
for (i in 0 until count) {
|
|
238
|
+
val format = extractor.getTrackFormat(i)
|
|
239
|
+
val track = JSONObject()
|
|
240
|
+
.put("index", i)
|
|
241
|
+
.put("mime", mimes[i])
|
|
242
|
+
.put(
|
|
243
|
+
"formatDurationUs",
|
|
244
|
+
if (format.containsKey(android.media.MediaFormat.KEY_DURATION)) {
|
|
245
|
+
format.getLong(android.media.MediaFormat.KEY_DURATION)
|
|
246
|
+
} else {
|
|
247
|
+
-1L
|
|
248
|
+
},
|
|
249
|
+
)
|
|
250
|
+
.put("sampleCount", counts[i])
|
|
251
|
+
.put("firstPtsUs", firsts[i])
|
|
252
|
+
.put("lastPtsUs", lasts[i])
|
|
253
|
+
videoTimes[i]?.let { track.put("timestampsUs", it) }
|
|
254
|
+
tracks.put(track)
|
|
255
|
+
}
|
|
256
|
+
out.put("tracks", tracks).put("ok", true)
|
|
257
|
+
} catch (e: Exception) {
|
|
258
|
+
out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
|
|
259
|
+
} finally {
|
|
260
|
+
runCatching { extractor.release() }
|
|
261
|
+
}
|
|
262
|
+
return out.toString()
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* A checksum per compressed video sample, without decoding.
|
|
267
|
+
*
|
|
268
|
+
* Transmuxing must move the encoded bytes and change only timestamps and
|
|
269
|
+
* container metadata. Comparing decoded pixels cannot show that: a re-encode
|
|
270
|
+
* that happens to look identical would pass, and this gate exists to tell
|
|
271
|
+
* those apart. So the payload itself is checksummed.
|
|
272
|
+
*/
|
|
273
|
+
fun sampleHashes(context: Context, uri: String): String {
|
|
274
|
+
val out = JSONObject()
|
|
275
|
+
val extractor = android.media.MediaExtractor()
|
|
276
|
+
try {
|
|
277
|
+
if (uri.startsWith("/")) {
|
|
278
|
+
extractor.setDataSource(uri)
|
|
279
|
+
} else {
|
|
280
|
+
extractor.setDataSource(context, Uri.parse(uri), null)
|
|
281
|
+
}
|
|
282
|
+
var video = -1
|
|
283
|
+
for (i in 0 until extractor.trackCount) {
|
|
284
|
+
val mime = extractor.getTrackFormat(i).getString(android.media.MediaFormat.KEY_MIME) ?: ""
|
|
285
|
+
if (mime.startsWith("video/")) { video = i; break }
|
|
286
|
+
}
|
|
287
|
+
/*
|
|
288
|
+
An audio-only file is a legitimate input here.
|
|
289
|
+
|
|
290
|
+
The audio timeline intermediate has no video track, and refusing it
|
|
291
|
+
made the payload comparison unreadable — it reported "no audio
|
|
292
|
+
hashes" for the one file that was nothing but audio.
|
|
293
|
+
*/
|
|
294
|
+
/*
|
|
295
|
+
One interleaved pass over every track.
|
|
296
|
+
|
|
297
|
+
Selecting and seeking per track reads nothing when a track's first
|
|
298
|
+
sample has a negative timestamp, which AAC's encoder delay routinely
|
|
299
|
+
produces -- the audio hashes came back empty for exactly the files
|
|
300
|
+
that had audio. Reading the stream as the extractor orders it and
|
|
301
|
+
asking which track each sample belongs to cannot make that mistake.
|
|
302
|
+
*/
|
|
303
|
+
for (i in 0 until extractor.trackCount) extractor.selectTrack(i)
|
|
304
|
+
extractor.seekTo(0, android.media.MediaExtractor.SEEK_TO_PREVIOUS_SYNC)
|
|
305
|
+
val buffer = java.nio.ByteBuffer.allocate(4 * 1024 * 1024)
|
|
306
|
+
|
|
307
|
+
val hashes = JSONArray()
|
|
308
|
+
val sizes = JSONArray()
|
|
309
|
+
val times = JSONArray()
|
|
310
|
+
val audioHashes = JSONArray()
|
|
311
|
+
val audioSizes = JSONArray()
|
|
312
|
+
val audioTimes = JSONArray()
|
|
313
|
+
val whole = java.util.zip.CRC32()
|
|
314
|
+
val beatEpoch = ExportHeartbeat.currentEpoch()
|
|
315
|
+
var beatCounter = 0
|
|
316
|
+
while (true) {
|
|
317
|
+
val track = extractor.sampleTrackIndex
|
|
318
|
+
if (track < 0) break
|
|
319
|
+
val t = extractor.sampleTime
|
|
320
|
+
buffer.clear()
|
|
321
|
+
val read = extractor.readSampleData(buffer, 0)
|
|
322
|
+
// Provenance hashing reads the whole file, and measured at
|
|
323
|
+
// 0.1x it was the longest unbeaten stretch of an entire export
|
|
324
|
+
// — 5.5 s of silence in a pipeline that was working perfectly.
|
|
325
|
+
// Conformance work beats too, so a gate run does not look
|
|
326
|
+
// stalled — but production does not run it at all, so the
|
|
327
|
+
// threshold is not measured against it.
|
|
328
|
+
beatCounter += 1
|
|
329
|
+
if (beatCounter % 64 == 0) ExportHeartbeat.beat("muxBatch", beatEpoch)
|
|
330
|
+
if (read <= 0) break
|
|
331
|
+
val bytes = ByteArray(read)
|
|
332
|
+
buffer.position(0)
|
|
333
|
+
buffer.get(bytes, 0, read)
|
|
334
|
+
val crc = java.util.zip.CRC32()
|
|
335
|
+
crc.update(bytes, 0, read)
|
|
336
|
+
if (track == video) {
|
|
337
|
+
whole.update(bytes, 0, read)
|
|
338
|
+
hashes.put(crc.value)
|
|
339
|
+
sizes.put(read)
|
|
340
|
+
times.put(t)
|
|
341
|
+
} else {
|
|
342
|
+
audioHashes.put(crc.value)
|
|
343
|
+
audioSizes.put(read)
|
|
344
|
+
audioTimes.put(t)
|
|
345
|
+
}
|
|
346
|
+
if (!extractor.advance()) break
|
|
347
|
+
}
|
|
348
|
+
out.put("sampleHashes", hashes)
|
|
349
|
+
.put("sampleSizes", sizes)
|
|
350
|
+
.put("timestampsUs", times)
|
|
351
|
+
.put("audioHashes", audioHashes)
|
|
352
|
+
.put("audioSizes", audioSizes)
|
|
353
|
+
.put("audioTimestampsUs", audioTimes)
|
|
354
|
+
.put("payloadCrc", whole.value)
|
|
355
|
+
.put("ok", true)
|
|
356
|
+
} catch (e: Exception) {
|
|
357
|
+
out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
|
|
358
|
+
} finally {
|
|
359
|
+
runCatching { extractor.release() }
|
|
360
|
+
}
|
|
361
|
+
return out.toString()
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Geometry and colour, separated.
|
|
366
|
+
*
|
|
367
|
+
* A maximum channel delta is close to useless around a transformed edge: one
|
|
368
|
+
* row of pixels landing on the far side of a boundary between a bright source
|
|
369
|
+
* and a black background produces a delta of 255 while the picture is right.
|
|
370
|
+
* So the frame is split into a **mask** — where the transformed source landed
|
|
371
|
+
* — and the colour inside it, and each is measured on its own terms.
|
|
372
|
+
*
|
|
373
|
+
* Reports numbers only. Which of them constitute a pass is the caller's
|
|
374
|
+
* predeclared business.
|
|
375
|
+
*/
|
|
376
|
+
fun geometry(
|
|
377
|
+
context: Context,
|
|
378
|
+
leftUri: String,
|
|
379
|
+
leftUs: Long,
|
|
380
|
+
rightUri: String,
|
|
381
|
+
rightUs: Long,
|
|
382
|
+
maskThreshold: Int,
|
|
383
|
+
): String {
|
|
384
|
+
val out = JSONObject()
|
|
385
|
+
try {
|
|
386
|
+
Reader(context, leftUri).use { l ->
|
|
387
|
+
Reader(context, rightUri).use { r ->
|
|
388
|
+
val a = l.frameAt(leftUs)
|
|
389
|
+
val b = r.frameAt(rightUs)
|
|
390
|
+
if (a == null || b == null) return out.put("ok", false).put("error", "no frame").toString()
|
|
391
|
+
if (a.width != b.width || a.height != b.height) {
|
|
392
|
+
return out.put("ok", false)
|
|
393
|
+
.put("error", "size ${a.width}x${a.height} vs ${b.width}x${b.height}")
|
|
394
|
+
.toString()
|
|
395
|
+
}
|
|
396
|
+
val w = a.width
|
|
397
|
+
val h = a.height
|
|
398
|
+
val pa = IntArray(w * h)
|
|
399
|
+
val pb = IntArray(w * h)
|
|
400
|
+
a.getPixels(pa, 0, w, 0, 0, w, h)
|
|
401
|
+
b.getPixels(pb, 0, w, 0, 0, w, h)
|
|
402
|
+
a.recycle()
|
|
403
|
+
b.recycle()
|
|
404
|
+
|
|
405
|
+
val ma = BooleanArray(w * h)
|
|
406
|
+
val mb = BooleanArray(w * h)
|
|
407
|
+
var aBox = intArrayOf(w, h, -1, -1)
|
|
408
|
+
var bBox = intArrayOf(w, h, -1, -1)
|
|
409
|
+
var areaA = 0L
|
|
410
|
+
var areaB = 0L
|
|
411
|
+
for (y in 0 until h) {
|
|
412
|
+
for (x in 0 until w) {
|
|
413
|
+
val i = y * w + x
|
|
414
|
+
if (maskValue(pa[i]) > maskThreshold) {
|
|
415
|
+
ma[i] = true; areaA++
|
|
416
|
+
if (x < aBox[0]) aBox[0] = x
|
|
417
|
+
if (y < aBox[1]) aBox[1] = y
|
|
418
|
+
if (x > aBox[2]) aBox[2] = x
|
|
419
|
+
if (y > aBox[3]) aBox[3] = y
|
|
420
|
+
}
|
|
421
|
+
if (maskValue(pb[i]) > maskThreshold) {
|
|
422
|
+
mb[i] = true; areaB++
|
|
423
|
+
if (x < bBox[0]) bBox[0] = x
|
|
424
|
+
if (y < bBox[1]) bBox[1] = y
|
|
425
|
+
if (x > bBox[2]) bBox[2] = x
|
|
426
|
+
if (y > bBox[3]) bBox[3] = y
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
var inter = 0L
|
|
431
|
+
var union = 0L
|
|
432
|
+
for (i in 0 until w * h) {
|
|
433
|
+
if (ma[i] && mb[i]) inter++
|
|
434
|
+
if (ma[i] || mb[i]) union++
|
|
435
|
+
}
|
|
436
|
+
/*
|
|
437
|
+
An empty union is not a perfect match.
|
|
438
|
+
|
|
439
|
+
Two frames that both drew nothing agree completely and mean
|
|
440
|
+
nothing, and IoU over an empty set is whatever the guard
|
|
441
|
+
says it is. Four transform cases once "passed" at 1000
|
|
442
|
+
permille while rendering black, which is the shape of
|
|
443
|
+
failure this refuses to report as success.
|
|
444
|
+
*/
|
|
445
|
+
if (union == 0L) {
|
|
446
|
+
return out.put("ok", false)
|
|
447
|
+
.put("error", "empty union: neither frame drew a footprint")
|
|
448
|
+
.put("maskAreaLeft", areaA).put("maskAreaRight", areaB)
|
|
449
|
+
.toString()
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/*
|
|
453
|
+
Interior only: the intersection, eroded by one pixel.
|
|
454
|
+
|
|
455
|
+
An edge pixel is a blend of source and background whose
|
|
456
|
+
value depends on exactly where the boundary fell, so it
|
|
457
|
+
reports colour error for what is really sub-pixel geometry.
|
|
458
|
+
Eroding separates the two questions instead of letting one
|
|
459
|
+
answer the other.
|
|
460
|
+
*/
|
|
461
|
+
var interiorPixels = 0L
|
|
462
|
+
var sumErr = 0L
|
|
463
|
+
var maxErr = 0
|
|
464
|
+
var differing = 0L
|
|
465
|
+
for (y in 1 until h - 1) {
|
|
466
|
+
for (x in 1 until w - 1) {
|
|
467
|
+
val i = y * w + x
|
|
468
|
+
if (!(ma[i] && mb[i])) continue
|
|
469
|
+
if (!(ma[i - 1] && ma[i + 1] && ma[i - w] && ma[i + w])) continue
|
|
470
|
+
if (!(mb[i - 1] && mb[i + 1] && mb[i - w] && mb[i + w])) continue
|
|
471
|
+
val p = pa[i]
|
|
472
|
+
val q = pb[i]
|
|
473
|
+
val dr = abs(((p shr 16) and 0xff) - ((q shr 16) and 0xff))
|
|
474
|
+
val dg = abs(((p shr 8) and 0xff) - ((q shr 8) and 0xff))
|
|
475
|
+
val db = abs((p and 0xff) - (q and 0xff))
|
|
476
|
+
val worst = maxOf(dr, dg, db)
|
|
477
|
+
if (worst > maxErr) maxErr = worst
|
|
478
|
+
sumErr += (dr + dg + db).toLong()
|
|
479
|
+
if (worst > 8) differing++
|
|
480
|
+
interiorPixels++
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
// Edge displacement, as how far the footprints' bounds moved.
|
|
485
|
+
var edge = 0
|
|
486
|
+
if (aBox[2] >= 0 && bBox[2] >= 0) {
|
|
487
|
+
for (k in 0 until 4) edge = maxOf(edge, abs(aBox[k] - bBox[k]))
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
out.put("ok", true)
|
|
491
|
+
.put("width", w).put("height", h)
|
|
492
|
+
.put("maskAreaLeft", areaA).put("maskAreaRight", areaB)
|
|
493
|
+
.put("intersection", inter).put("union", union)
|
|
494
|
+
.put("iouPermille", if (union == 0L) 1000 else ((inter * 1000) / union).toInt())
|
|
495
|
+
.put("symmetricDifference", union - inter)
|
|
496
|
+
.put("bboxLeft", JSONArray().put(aBox[0]).put(aBox[1]).put(aBox[2]).put(aBox[3]))
|
|
497
|
+
.put("bboxRight", JSONArray().put(bBox[0]).put(bBox[1]).put(bBox[2]).put(bBox[3]))
|
|
498
|
+
.put("edgeDisplacementPx", edge)
|
|
499
|
+
.put("interiorPixels", interiorPixels)
|
|
500
|
+
.put("interiorMeanError", if (interiorPixels == 0L) 0.0 else sumErr.toDouble() / (interiorPixels * 3))
|
|
501
|
+
.put("interiorMaxError", maxErr)
|
|
502
|
+
.put(
|
|
503
|
+
"interiorDifferingPermille",
|
|
504
|
+
if (interiorPixels == 0L) 0 else ((differing * 1000) / interiorPixels).toInt(),
|
|
505
|
+
)
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
} catch (e: Exception) {
|
|
509
|
+
out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
|
|
510
|
+
}
|
|
511
|
+
return out.toString()
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/** One decoded frame as base64 PNG, in display space. */
|
|
515
|
+
fun framePng(context: Context, uri: String, timeUs: Long): String {
|
|
516
|
+
val out = JSONObject()
|
|
517
|
+
try {
|
|
518
|
+
Reader(context, uri).use { reader ->
|
|
519
|
+
val bitmap = reader.frameAt(timeUs)
|
|
520
|
+
?: return out.put("ok", false).put("error", "no frame").toString()
|
|
521
|
+
val stream = java.io.ByteArrayOutputStream()
|
|
522
|
+
bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
|
|
523
|
+
out.put("ok", true)
|
|
524
|
+
.put("width", bitmap.width)
|
|
525
|
+
.put("height", bitmap.height)
|
|
526
|
+
.put("png", android.util.Base64.encodeToString(stream.toByteArray(), android.util.Base64.NO_WRAP))
|
|
527
|
+
bitmap.recycle()
|
|
528
|
+
}
|
|
529
|
+
} catch (e: Exception) {
|
|
530
|
+
out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
|
|
531
|
+
}
|
|
532
|
+
return out.toString()
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* Mask test: the brightest channel, not luminance.
|
|
537
|
+
*
|
|
538
|
+
* Luminance weights blue at 11%, so a blue-only source at value 93 scores 13
|
|
539
|
+
* — indistinguishable from black at any usable threshold, and the mask
|
|
540
|
+
* collapsed to the burned-in text instead of the source's footprint. The
|
|
541
|
+
* question here is "is there source here at all", and the strongest channel
|
|
542
|
+
* answers it without caring which channel carries the signal.
|
|
543
|
+
*/
|
|
544
|
+
/**
|
|
545
|
+
* Mean and extremes over caller-chosen rectangles.
|
|
546
|
+
*
|
|
547
|
+
* The fixed centre region [sample] uses is wrong for a transformed clip: it
|
|
548
|
+
* straddles the footprint edge, so it averages source pixels with cleared
|
|
549
|
+
* ones and the recovered source frame reads low. The caller knows where the
|
|
550
|
+
* footprint is — it has the corner oracle — so it passes the rectangles and
|
|
551
|
+
* this reports what is in them.
|
|
552
|
+
*
|
|
553
|
+
* `max - min` per channel is returned as well as the mean, because a region
|
|
554
|
+
* that is supposed to be uniform and is not is a different failure from one
|
|
555
|
+
* whose average is off.
|
|
556
|
+
*/
|
|
557
|
+
fun probe(context: Context, uri: String, probesJson: String): String {
|
|
558
|
+
val out = JSONObject()
|
|
559
|
+
try {
|
|
560
|
+
val probes = JSONArray(probesJson)
|
|
561
|
+
Reader(context, uri).use { reader ->
|
|
562
|
+
val results = JSONArray()
|
|
563
|
+
var lastUs = Long.MIN_VALUE
|
|
564
|
+
var bitmap: Bitmap? = null
|
|
565
|
+
for (i in 0 until probes.length()) {
|
|
566
|
+
val p = probes.getJSONObject(i)
|
|
567
|
+
val us = p.getLong("timeUs")
|
|
568
|
+
if (us != lastUs) {
|
|
569
|
+
bitmap?.recycle()
|
|
570
|
+
bitmap = reader.frameAt(us)
|
|
571
|
+
lastUs = us
|
|
572
|
+
}
|
|
573
|
+
val bm = bitmap
|
|
574
|
+
val entry = JSONObject().put("timeUs", us)
|
|
575
|
+
.put("x", p.getInt("x")).put("y", p.getInt("y"))
|
|
576
|
+
.put("w", p.getInt("w")).put("h", p.getInt("h"))
|
|
577
|
+
if (bm == null) {
|
|
578
|
+
results.put(entry.put("error", "no frame")); continue
|
|
579
|
+
}
|
|
580
|
+
val x0 = p.getInt("x").coerceIn(0, bm.width - 1)
|
|
581
|
+
val y0 = p.getInt("y").coerceIn(0, bm.height - 1)
|
|
582
|
+
val w = p.getInt("w").coerceIn(1, bm.width - x0)
|
|
583
|
+
val h = p.getInt("h").coerceIn(1, bm.height - y0)
|
|
584
|
+
val row = IntArray(w)
|
|
585
|
+
var r = 0L; var g = 0L; var b = 0L
|
|
586
|
+
var rMin = 255; var gMin = 255; var bMin = 255
|
|
587
|
+
var rMax = 0; var gMax = 0; var bMax = 0
|
|
588
|
+
for (y in y0 until y0 + h) {
|
|
589
|
+
bm.getPixels(row, 0, w, x0, y, w, 1)
|
|
590
|
+
for (px in row) {
|
|
591
|
+
val pr = (px shr 16) and 0xff
|
|
592
|
+
val pg = (px shr 8) and 0xff
|
|
593
|
+
val pb = px and 0xff
|
|
594
|
+
r += pr; g += pg; b += pb
|
|
595
|
+
if (pr < rMin) rMin = pr
|
|
596
|
+
if (pg < gMin) gMin = pg
|
|
597
|
+
if (pb < bMin) bMin = pb
|
|
598
|
+
if (pr > rMax) rMax = pr
|
|
599
|
+
if (pg > gMax) gMax = pg
|
|
600
|
+
if (pb > bMax) bMax = pb
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
val n = w.toLong() * h.toLong()
|
|
604
|
+
results.put(
|
|
605
|
+
entry.put("r", r.toDouble() / n).put("g", g.toDouble() / n).put("b", b.toDouble() / n)
|
|
606
|
+
.put("rMin", rMin).put("gMin", gMin).put("bMin", bMin)
|
|
607
|
+
.put("rMax", rMax).put("gMax", gMax).put("bMax", bMax)
|
|
608
|
+
.put("pixels", n),
|
|
609
|
+
)
|
|
610
|
+
}
|
|
611
|
+
bitmap?.recycle()
|
|
612
|
+
out.put("probes", results).put("ok", true)
|
|
613
|
+
}
|
|
614
|
+
} catch (e: Exception) {
|
|
615
|
+
out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
|
|
616
|
+
}
|
|
617
|
+
return out.toString()
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
private fun maskValue(p: Int): Int {
|
|
621
|
+
val r = (p shr 16) and 0xff
|
|
622
|
+
val g = (p shr 8) and 0xff
|
|
623
|
+
val b = p and 0xff
|
|
624
|
+
return maxOf(r, maxOf(g, b))
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
private class Stats(val maxDelta: Int, val meanDelta: Double, val differingPermille: Int)
|
|
628
|
+
|
|
629
|
+
private fun diff(a: Bitmap, b: Bitmap): Stats {
|
|
630
|
+
val w = a.width
|
|
631
|
+
val h = a.height
|
|
632
|
+
val rowA = IntArray(w)
|
|
633
|
+
val rowB = IntArray(w)
|
|
634
|
+
var max = 0
|
|
635
|
+
var sum = 0L
|
|
636
|
+
var differing = 0L
|
|
637
|
+
var total = 0L
|
|
638
|
+
for (y in 0 until h) {
|
|
639
|
+
a.getPixels(rowA, 0, w, 0, y, w, 1)
|
|
640
|
+
b.getPixels(rowB, 0, w, 0, y, w, 1)
|
|
641
|
+
for (x in 0 until w) {
|
|
642
|
+
val p = rowA[x]
|
|
643
|
+
val q = rowB[x]
|
|
644
|
+
val dr = abs(((p shr 16) and 0xff) - ((q shr 16) and 0xff))
|
|
645
|
+
val dg = abs(((p shr 8) and 0xff) - ((q shr 8) and 0xff))
|
|
646
|
+
val db = abs((p and 0xff) - (q and 0xff))
|
|
647
|
+
if (dr > max) max = dr
|
|
648
|
+
if (dg > max) max = dg
|
|
649
|
+
if (db > max) max = db
|
|
650
|
+
sum += dr + dg + db
|
|
651
|
+
if (dr != 0 || dg != 0 || db != 0) differing += 1
|
|
652
|
+
total += 3
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
val pixels = w.toLong() * h.toLong()
|
|
656
|
+
return Stats(
|
|
657
|
+
maxDelta = max,
|
|
658
|
+
meanDelta = if (total == 0L) 0.0 else sum.toDouble() / total.toDouble(),
|
|
659
|
+
differingPermille = if (pixels == 0L) 0 else ((differing * 1000) / pixels).toInt(),
|
|
660
|
+
)
|
|
661
|
+
}
|
|
662
|
+
}
|