@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/Diagnostics.kt

@@ -0,0 +1,54 @@
+package dev.pictor.video
+
+import org.json.JSONObject
+
+/**
+ * Whether gate-only behaviour exists in this build.
+ *
+ * ## Why this is not a runtime flag
+ *
+ * It was one. `ExportGate.diagnostics` was a `@Volatile var` that a host armed
+ * by calling a method — which meant a shipped build was one call away from
+ * being able to inject a cadence failure, wedge an export or spend a user's
+ * battery hashing every sample of every intermediate. "The production exporter
+ * never calls it" is a statement about one file, not about the binary, and the
+ * requirement is about the binary.
+ *
+ * So the decision moved to compile time. [ENABLED] is a `BuildConfig` constant:
+ * true in debug, true in a release built with `-PpictorDiagnostics=true`, and
+ * false in every other release. No sequence of calls from JavaScript or from
+ * native changes it, because there is nothing to change.
+ *
+ * ## What it guards
+ *
+ * Two things, and both are needed. Every **mutating** gate method refuses at its
+ * first line, so arming is impossible; and every site that would **act** on an
+ * injected fault is also guarded, so a value that somehow got set could still
+ * not do anything. The second is what makes this a boundary rather than a lock
+ * on one door.
+ *
+ * ## What it does not guard
+ *
+ * Read-only accounting — the run and resource ledgers, heartbeat counts, the
+ * live-temporary list — stays available in release. It is how cleanup is proven
+ * rather than asserted, it changes no execution path, and separating it from the
+ * fault controls is the point: a diagnostic that alters behaviour is not a
+ * diagnostic.
+ */
+internal object Diagnostics {
+    /** Compile-time. Nothing at runtime can change it. */
+    val ENABLED: Boolean = BuildConfig.PICTOR_DIAGNOSTICS
+
+    /**
+     * The one refusal every mutating gate method returns in a release build.
+     *
+     * Stable and synchronous: a caller learns immediately that the control does
+     * not exist here, rather than believing it armed something.
+     */
+    fun unavailable(): String =
+        JSONObject()
+            .put("ok", false)
+            .put("diagnostics", false)
+            .put("error", "diagnostics unavailable in this build")
+            .toString()
+}

package/android/src/main/java/dev/pictor/video/EncoderGate.kt

@@ -0,0 +1,476 @@
+package dev.pictor.video
+
+import android.content.Context
+import android.media.MediaCodec
+import android.media.MediaExtractor
+import android.media.MediaFormat
+import java.io.File
+import java.nio.ByteBuffer
+import java.util.zip.CRC32
+import kotlin.math.abs
+import kotlin.math.min
+import kotlin.math.sin
+import org.json.JSONArray
+import org.json.JSONObject
+
+/**
+ * Freezing what the AAC encoder does, before anything changes it.
+ *
+ * The encode is 89% of a preview preparation and the loop that drives it is
+ * about to be rewritten. A rewrite of a codec loop can go wrong in ways nothing
+ * downstream notices — a dropped access unit, a timestamp that restarts, a
+ * sample shifted by a frame — so the shape of the output is recorded first, in
+ * enough detail that "the same" can be checked rather than assumed.
+ *
+ * ## Inputs are generated, not read
+ *
+ * Every fixture is synthesised here from a formula. A file could be replaced,
+ * re-encoded or reclaimed between the two halves of the comparison; an
+ * arithmetic sequence cannot. That is what makes the baseline immutable.
+ *
+ * ## What is recorded
+ *
+ * Both the compressed shape — access-unit count, timestamps, CSD, per-unit
+ * payload hashes, bytes — and the decoded result — length, RMS windows, and the
+ * positions of transients. Payload equality is the strong claim; if scheduling
+ * changes how samples are packaged into access units, the decoded comparison is
+ * what still has to hold.
+ */
+object EncoderGate {
+
+    private const val RATE = AudioMixer.SAMPLE_RATE
+    private const val CHANNELS = AudioMixer.CHANNELS
+
+    /** Marker positions, in seconds, for the transient fixture. */
+    private val MARKERS_S = doubleArrayOf(0.0, 1.0, 2.5, 4.9)
+
+    /**
+     * The fixtures, by name.
+     *
+     * Deliberately spanning what the encoder has to survive: a trivial length,
+     * the acceptance length, pure silence, content that saturates, and sharp
+     * transients whose positions can be found again after a round trip.
+     */
+    fun pcmFor(name: String): FloatArray = when (name) {
+        "short" -> tone(1.0, 440.0, 0.5)
+        "acceptance" -> tone(33.0, 220.0, 0.4)
+        "silence" -> FloatArray((2.0 * RATE).toInt() * CHANNELS)
+        "saturated" -> saturated(2.0)
+        "markers" -> markers(5.0)
+        else -> FloatArray(0)
+    }
+
+    private fun tone(seconds: Double, hz: Double, amplitude: Double): FloatArray {
+        val frames = (seconds * RATE).toInt()
+        val out = FloatArray(frames * CHANNELS)
+        for (i in 0 until frames) {
+            val v = (amplitude * sin(2.0 * Math.PI * hz * i / RATE)).toFloat()
+            for (c in 0 until CHANNELS) out[i * CHANNELS + c] = v
+        }
+        return out
+    }
+
+    /** Two tones summed past full scale, so the saturation clamp is exercised. */
+    private fun saturated(seconds: Double): FloatArray {
+        val frames = (seconds * RATE).toInt()
+        val out = FloatArray(frames * CHANNELS)
+        for (i in 0 until frames) {
+            val a = 0.8 * sin(2.0 * Math.PI * 440.0 * i / RATE)
+            val b = 0.8 * sin(2.0 * Math.PI * 660.0 * i / RATE)
+            val v = maxOf(-1.0, min(1.0, a + b)).toFloat()
+            for (c in 0 until CHANNELS) out[i * CHANNELS + c] = v
+        }
+        return out
+    }
+
+    /** Silence with one-millisecond full-scale bursts at known times. */
+    private fun markers(seconds: Double): FloatArray {
+        val frames = (seconds * RATE).toInt()
+        val out = FloatArray(frames * CHANNELS)
+        val burst = (RATE / 1000)
+        for (at in MARKERS_S) {
+            val start = (at * RATE).toInt()
+            for (i in start until min(frames, start + burst)) {
+                // Alternating full scale: broadband, and unmistakable against
+                // silence after a lossy round trip.
+                val v = if ((i - start) % 2 == 0) 0.95f else -0.95f
+                for (c in 0 until CHANNELS) out[i * CHANNELS + c] = v
+            }
+        }
+        return out
+    }
+
+    /**
+     * Write a fixture out as a real audio file, so a mix can use it as a source.
+     *
+     * The alignment gate needs a source whose content is known to the
+     * microsecond. Generating it here rather than shipping one keeps the
+     * marker positions a property of the code that checks them.
+     */
+    fun writeFixture(context: Context, name: String, into: File): JSONObject {
+        if (!Diagnostics.ENABLED) return JSONObject(Diagnostics.unavailable())
+        val pcm = pcmFor(name)
+        if (pcm.isEmpty()) return JSONObject().put("ok", false).put("error", "unknown fixture")
+        val encoded = AudioMixer.encodeForGate(pcm, into, 65536)
+        if (!encoded.optBoolean("ok", false)) return encoded
+        return JSONObject()
+            .put("ok", true)
+            .put("uri", android.net.Uri.fromFile(into).toString())
+            .put("frames", pcm.size / CHANNELS)
+            .put("durationUs", (pcm.size / CHANNELS).toLong() * 1_000_000L / RATE)
+            .put("markersMs", JSONArray().also { a -> MARKERS_S.forEach { a.put((it * 1000).toInt()) } })
+    }
+
+    /**
+     * Write a fixture as uncompressed WAV.
+     *
+     * The alignment gate used an AAC source and measured +85 ms, which split
+     * into two equal halves: the mix's own priming, and the priming of the
+     * source the gate had itself encoded. The second half was an artifact of
+     * the measurement — so the source stops being compressed. WAV has no
+     * priming, no delay and no encoder, which makes the marker positions in the
+     * file exactly the marker positions in the formula.
+     */
+    fun writeWavFixture(context: Context, name: String, into: File): JSONObject {
+        if (!Diagnostics.ENABLED) return JSONObject(Diagnostics.unavailable())
+        val pcm = pcmFor(name)
+        if (pcm.isEmpty()) return JSONObject().put("ok", false).put("error", "unknown fixture")
+
+        val frames = pcm.size / CHANNELS
+        val dataBytes = pcm.size * 2
+        val header = java.nio.ByteBuffer.allocate(44).order(java.nio.ByteOrder.LITTLE_ENDIAN)
+        header.put("RIFF".toByteArray())
+        header.putInt(36 + dataBytes)
+        header.put("WAVE".toByteArray())
+        header.put("fmt ".toByteArray())
+        header.putInt(16)
+        header.putShort(1) // PCM
+        header.putShort(CHANNELS.toShort())
+        header.putInt(RATE)
+        header.putInt(RATE * CHANNELS * 2) // byte rate
+        header.putShort((CHANNELS * 2).toShort()) // block align
+        header.putShort(16) // bits per sample
+        header.put("data".toByteArray())
+        header.putInt(dataBytes)
+
+        java.io.FileOutputStream(into).use { file ->
+            file.write(header.array())
+            val body = java.nio.ByteBuffer.allocate(dataBytes).order(java.nio.ByteOrder.LITTLE_ENDIAN)
+            for (v in pcm) body.putShort((v * 32767f).toInt().toShort())
+            file.write(body.array())
+            file.flush()
+            file.fd.sync()
+        }
+
+        return JSONObject()
+            .put("ok", true)
+            .put("uri", android.net.Uri.fromFile(into).toString())
+            .put("frames", frames)
+            .put("durationUs", frames.toLong() * 1_000_000L / RATE)
+            .put("markersMs", JSONArray().also { a -> MARKERS_S.forEach { a.put((it * 1000).toInt()) } })
+    }
+
+    /**
+     * Where the markers ended up in a finished mix, against where they belong.
+     *
+     * This is the question the raw encoder baseline cannot answer. That baseline
+     * showed a constant +42 ms in the encoded file, which is AAC priming and is
+     * expected — the export removes it at the mux by dropping whole access
+     * units. What matters for preview is whether the file a player is handed is
+     * aligned, and the only way to know is to decode what was produced and look.
+     */
+    fun alignment(context: Context, mixFile: File, expectedMs: JSONArray): JSONObject {
+        val decoded = decode(mixFile)
+        val found = markerPositions(decoded)
+        val offsets = JSONArray()
+        var worst = 0
+        for (i in 0 until minOf(found.length(), expectedMs.length())) {
+            val d = found.getInt(i) - expectedMs.getInt(i)
+            offsets.put(d)
+            if (abs(d.toDouble()) > abs(worst.toDouble())) worst = d
+        }
+        /*
+          The head, measured rather than thresholded.
+
+          A transient at timeline zero has no silence in front of it once the
+          priming is removed, and AAC's overlapping transform spreads its energy
+          across the frames either side — including into the priming units that
+          get dropped. So "the onset detector did not fire" and "the sound is
+          gone" are different claims, and only the second one matters. This
+          reports what is actually in the first few milliseconds.
+        */
+        val head = JSONObject()
+        for (ms in intArrayOf(5, 20, 60)) {
+            var peak = 0.0
+            val upto = minOf(decoded.size, ms * RATE / 1000 * CHANNELS)
+            for (i in 0 until upto) peak = maxOf(peak, abs(decoded[i].toDouble()))
+            head.put("peakFirst${ms}ms", Math.round(peak * 1000.0) / 1000.0)
+        }
+
+        return JSONObject()
+            .put("ok", true)
+            .put("decodedFrames", decoded.size / CHANNELS)
+            .put("foundMs", found)
+            .put("expectedMs", expectedMs)
+            .put("offsetsMs", offsets)
+            .put("worstOffsetMs", worst)
+            .put("head", head)
+    }
+
+    /** Decode a file to interleaved float PCM. Gate use. */
+    fun decodeFile(file: File): FloatArray = decode(file)
+
+    /**
+     * The five places a sample can go missing, measured in one run.
+     *
+     * Mixing, encoder delay, decoder startup and mux trimming all look the same
+     * from the outside — a marker that is not where it should be — so each is
+     * measured separately against the same deterministic source.
+     */
+    fun describeStage(name: String, pcm: FloatArray, offsetMs: Int = 0): JSONObject {
+        val frames = pcm.size / CHANNELS
+        var peak60 = 0.0
+        val upto = minOf(pcm.size, 60 * RATE / 1000 * CHANNELS)
+        for (i in 0 until upto) peak60 = maxOf(peak60, abs(pcm[i].toDouble()))
+        val found = markerPositions(pcm)
+        val shifted = JSONArray()
+        for (i in 0 until found.length()) shifted.put(found.getInt(i) + offsetMs)
+        return JSONObject()
+            .put("stage", name)
+            .put("frames", frames)
+            .put("durationMs", frames.toLong() * 1000 / RATE)
+            .put("peakFirst60ms", Math.round(peak60 * 1000.0) / 1000.0)
+            .put("markersMs", shifted)
+            .put("rms", Math.round(rms(pcm) * 100000.0) / 100000.0)
+    }
+
+    /** The fixture's own marker positions, for comparison. */
+    fun expectedMarkersMs(): JSONArray =
+        JSONArray().also { a -> MARKERS_S.forEach { a.put((it * 1000).toInt()) } }
+
+    /**
+     * Encode one fixture and describe the result completely.
+     *
+     * Gate-only: it writes a temporary file, measures it and removes it.
+     */
+    fun run(context: Context, name: String, maxInputSize: Int): JSONObject {
+        if (!Diagnostics.ENABLED) return JSONObject(Diagnostics.unavailable())
+        val pcm = pcmFor(name)
+        if (pcm.isEmpty()) {
+            return JSONObject().put("ok", false).put("error", "unknown fixture '$name'")
+        }
+
+        val file = File.createTempFile("pictor-encgate-", ".m4a", context.cacheDir)
+        return try {
+            val at = System.nanoTime()
+            val encoded = AudioMixer.encodeForGate(pcm, file, maxInputSize)
+            val encodeMs = (System.nanoTime() - at) / 1_000_000
+            if (!encoded.optBoolean("ok", false)) {
+                JSONObject().put("ok", false).put("error", encoded.optString("error"))
+            } else {
+                describe(context, file, pcm, name)
+                    .put("ok", true)
+                    .put("fixture", name)
+                    .put("inputFrames", pcm.size / CHANNELS)
+                    .put("inputSamples", pcm.size)
+                    .put("encodeMs", encodeMs)
+                    .put("maxInputSize", maxInputSize)
+                    .put("fileBytes", file.length())
+            }
+        } catch (e: Throwable) {
+            JSONObject().put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
+        } finally {
+            runCatching { file.delete() }
+        }
+    }
+
+    /** The compressed shape, and the decoded result. */
+    private fun describe(context: Context, file: File, source: FloatArray, name: String): JSONObject {
+        val out = JSONObject()
+        val extractor = MediaExtractor()
+        extractor.setDataSource(file.absolutePath)
+        var track = -1
+        for (i in 0 until extractor.trackCount) {
+            val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
+            if (mime.startsWith("audio/")) { track = i; break }
+        }
+        if (track < 0) {
+            extractor.release()
+            return out.put("units", 0).put("error", "no audio track")
+        }
+
+        val format = extractor.getTrackFormat(track)
+        out.put("mime", format.getString(MediaFormat.KEY_MIME))
+        out.put("trackRate", format.getInteger(MediaFormat.KEY_SAMPLE_RATE))
+        out.put("trackChannels", format.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
+        if (format.containsKey(MediaFormat.KEY_DURATION)) {
+            out.put("trackDurationUs", format.getLong(MediaFormat.KEY_DURATION))
+        }
+        // The codec-specific data, hashed. A rewrite that changed it would
+        // produce a file older readers could not open.
+        format.getByteBuffer("csd-0")?.let { out.put("csd0", hashOf(it.duplicate())) }
+
+        extractor.selectTrack(track)
+        val buffer = ByteBuffer.allocate(256 * 1024)
+        val timestamps = JSONArray()
+        val payloads = CRC32()
+        var units = 0
+        var bytes = 0L
+        var firstUs = -1L
+        var lastUs = -1L
+        while (true) {
+            val size = extractor.readSampleData(buffer, 0)
+            if (size < 0) break
+            val us = extractor.sampleTime
+            if (firstUs < 0) firstUs = us
+            lastUs = us
+            // Every timestamp for the first few units, then a rolling hash: the
+            // head is where a restart shows, and the hash catches the rest.
+            if (units < 8) timestamps.put(us)
+            val copy = ByteArray(size)
+            buffer.position(0)
+            buffer.get(copy, 0, size)
+            payloads.update(copy)
+            units += 1
+            bytes += size
+            buffer.clear()
+            if (!extractor.advance()) break
+        }
+        extractor.release()
+
+        out.put("units", units)
+            .put("payloadBytes", bytes)
+            .put("payloadHash", payloads.value)
+            .put("firstUs", firstUs)
+            .put("lastUs", lastUs)
+            .put("headTimestamps", timestamps)
+
+        /*
+          Decoded back, because payload equality is the strong claim and this is
+          the one that still has to hold if scheduling repackages the units.
+        */
+        val decoded = decode(file)
+        out.put("decodedFrames", decoded.size / CHANNELS)
+        out.put("decodedRms", rms(decoded))
+        out.put("sourceRms", rms(source))
+        out.put("peak", peak(decoded))
+        if (name == "markers") {
+            out.put("markerPositionsMs", markerPositions(decoded))
+            out.put("expectedMarkersMs", JSONArray().also { a -> MARKERS_S.forEach { a.put((it * 1000).toInt()) } })
+        }
+        return out
+    }
+
+    /** Decode the encoded file back to interleaved float PCM. */
+    private fun decode(file: File): FloatArray {
+        val extractor = MediaExtractor()
+        extractor.setDataSource(file.absolutePath)
+        var track = -1
+        for (i in 0 until extractor.trackCount) {
+            val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
+            if (mime.startsWith("audio/")) { track = i; break }
+        }
+        if (track < 0) { extractor.release(); return FloatArray(0) }
+        extractor.selectTrack(track)
+        val format = extractor.getTrackFormat(track)
+        val codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!)
+        codec.configure(format, null, null, 0)
+        codec.start()
+
+        val raw = ArrayList<Float>()
+        val info = MediaCodec.BufferInfo()
+        var inputDone = false
+        var outputDone = false
+        var guard = 0
+        try {
+            while (!outputDone && guard < 500_000) {
+                guard += 1
+                if (!inputDone) {
+                    val index = codec.dequeueInputBuffer(5_000)
+                    if (index >= 0) {
+                        val buf = codec.getInputBuffer(index)!!
+                        val size = extractor.readSampleData(buf, 0)
+                        if (size < 0) {
+                            codec.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
+                            inputDone = true
+                        } else {
+                            codec.queueInputBuffer(index, 0, size, extractor.sampleTime, 0)
+                            extractor.advance()
+                        }
+                    }
+                }
+                val index = codec.dequeueOutputBuffer(info, 5_000)
+                if (index >= 0) {
+                    if (info.size > 0) {
+                        val buf = codec.getOutputBuffer(index)!!
+                        val shorts = buf.order(java.nio.ByteOrder.nativeOrder()).asShortBuffer()
+                        while (shorts.hasRemaining()) raw.add(shorts.get() / 32768f)
+                    }
+                    codec.releaseOutputBuffer(index, false)
+                    if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) outputDone = true
+                }
+            }
+        } finally {
+            runCatching { codec.stop() }
+            runCatching { codec.release() }
+            runCatching { extractor.release() }
+        }
+        return FloatArray(raw.size) { raw[it] }
+    }
+
+    private fun rms(pcm: FloatArray): Double {
+        if (pcm.isEmpty()) return 0.0
+        var sum = 0.0
+        for (v in pcm) sum += v.toDouble() * v
+        return Math.sqrt(sum / pcm.size)
+    }
+
+    private fun peak(pcm: FloatArray): Double {
+        var max = 0.0
+        for (v in pcm) { val a = abs(v.toDouble()); if (a > max) max = a }
+        return max
+    }
+
+    /**
+     * Where the transients ended up, in milliseconds.
+     *
+     * A lossy round trip smears a one-millisecond burst, so this looks for the
+     * onset — the first frame past a threshold after a quiet run — rather than
+     * a peak, which can move within the smear.
+     */
+    private fun markerPositions(pcm: FloatArray): JSONArray {
+        val out = JSONArray()
+        val frames = pcm.size / CHANNELS
+        /*
+          Starts "already quiet", so a transient at sample zero is found.
+
+          It started at zero and required a preceding quiet run, which meant a
+          marker at position zero was never reported — and that is precisely the
+          case the priming trim creates. The first corrected run looked like a
+          lost marker and was a blind detector: before the fix there were 42 ms
+          of priming silence in front of it to satisfy the condition.
+        */
+        var quiet = Int.MAX_VALUE / 2
+        var i = 0
+        while (i < frames) {
+            var level = 0.0
+            for (c in 0 until CHANNELS) level = maxOf(level, abs(pcm[i * CHANNELS + c].toDouble()))
+            if (level > 0.25) {
+                if (quiet > RATE / 50) out.put((i.toLong() * 1000 / RATE).toInt())
+                quiet = 0
+            } else {
+                quiet += 1
+            }
+            i += 1
+        }
+        return out
+    }
+
+    private fun hashOf(buffer: ByteBuffer): Long {
+        val crc = CRC32()
+        val copy = ByteArray(buffer.remaining())
+        buffer.get(copy)
+        crc.update(copy)
+        return crc.value
+    }
+}