@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/AacPriming.kt

@@ -0,0 +1,261 @@
+package dev.pictor.video
+
+import android.media.MediaCodec
+import android.media.MediaCodecInfo
+import android.media.MediaFormat
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import kotlin.math.abs
+import org.json.JSONObject
+
+/**
+ * How many samples of priming this encoder prepends — measured, not assumed.
+ *
+ * A lossy encoder emits its filter-bank warm-up before any content, so a track
+ * muxed without compensating plays late by that much. Measured on a Pixel 7: a
+ * uniform 42.7 ms A/V offset, well past a video frame at 30 fps, identical at
+ * every marker in a ten-second timeline.
+ *
+ * Three things this deliberately does **not** do.
+ *
+ * It does not read `KEY_ENCODER_DELAY`: this encoder leaves it unset, and a
+ * value that is absent on the device you have is not a value you can build on.
+ *
+ * It does not hard-code 2048. That is the number this encoder happens to use,
+ * and a constant that is right on one device is indistinguishable from a
+ * measurement until it reaches a device where it is wrong.
+ *
+ * It does not look for the first loud sample in the user's audio. A clip that
+ * legitimately begins with silence would then be diagnosed as heavily primed and
+ * trimmed into its own content. Delay is a property of the codec and its
+ * configuration, so it is measured from a signal whose position is already
+ * known, and cached against that configuration.
+ */
+internal object AacPriming {
+
+    /** Frames of calibration signal; long enough that the burst is unambiguous. */
+    private const val LENGTH = 16_384
+
+    /** The burst itself: short, full-scale, and easy to find after a codec. */
+    private const val BURST = 256
+
+    private val cache = HashMap<String, Result>()
+
+    /** What the measurement found, and how. */
+    class Result(
+        val delayFrames: Int,
+        /** `"timestamps"` or `"calibration"` — which method identified it. */
+        val method: String,
+        /** Onset the calibration observed, before subtracting the known position. */
+        val observedOnset: Int,
+        val expectedOnset: Int,
+    )
+
+    private fun key(rate: Int, channels: Int, bitrate: Int) = "$rate/$channels/$bitrate"
+
+    /**
+     * Measure (or recall) the priming for one encoder configuration.
+     *
+     * @param at where the calibration burst is placed. Callers use 0; the gate
+     *   also runs the middle and the final frames, because a correct measurement
+     *   must not depend on where the content sits.
+     */
+    fun measure(rate: Int, channels: Int, bitrate: Int, at: Int = 0, useCache: Boolean = true): Result {
+        val k = key(rate, channels, bitrate)
+        if (useCache && at == 0) cache[k]?.let { return it }
+
+        val pcm = ShortArray(LENGTH * channels)
+        for (i in 0 until BURST) {
+            val f = at + i
+            if (f >= LENGTH) break
+            // Alternating full scale: survives the codec as an unmistakable
+            // burst of energy, and a tone would ring past its own edge.
+            val v = if (i % 2 == 0) 30_000 else -30_000
+            for (c in 0 until channels) pcm[f * channels + c] = v.toShort()
+        }
+
+        val encoded = encode(pcm, rate, channels, bitrate)
+        /*
+          The timestamp relationship first, because it needs no signal at all.
+
+          If an encoder reports output timestamps shifted from the inputs that
+          produced them, that shift *is* the delay and is exact. Most Android AAC
+          encoders copy input timestamps straight through, in which case the
+          relationship carries no information and the calibration below has to
+          answer instead -- so the outcome is recorded either way rather than
+          assumed.
+        */
+        if (encoded.timestampShiftFrames > 0) {
+            val r = Result(encoded.timestampShiftFrames, "timestamps", -1, -1)
+            if (at == 0) cache[k] = r
+            return r
+        }
+
+        val decoded = decode(encoded.units, encoded.csd, rate, channels)
+        var peak = 0
+        for (v in decoded) if (abs(v.toInt()) > peak) peak = abs(v.toInt())
+        val threshold = (peak * 0.25).toInt().coerceAtLeast(1)
+        var onset = -1
+        var f = 0
+        while (f * channels < decoded.size) {
+            if (abs(decoded[f * channels].toInt()) >= threshold) { onset = f; break }
+            f += 1
+        }
+        if (onset < 0) throw IllegalStateException("calibration burst did not survive the encoder")
+        val r = Result(onset - at, "calibration", onset, at)
+        if (at == 0) cache[k] = r
+        return r
+    }
+
+    private class Encoded(
+        val units: List<ByteArray>,
+        val csd: ByteBuffer,
+        val timestampShiftFrames: Int,
+    )
+
+    private fun encode(pcm: ShortArray, rate: Int, channels: Int, bitrate: Int): Encoded {
+        val format = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, rate, channels).apply {
+            setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC)
+            setInteger(MediaFormat.KEY_BIT_RATE, bitrate)
+        }
+        val ledgerEntry = CodecLedger.open("priming.encoder")
+        val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
+        codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
+        codec.start()
+        val units = ArrayList<ByteArray>()
+        var csd: ByteBuffer? = null
+        val info = MediaCodec.BufferInfo()
+        var offset = 0
+        var inputDone = false
+        var outputDone = false
+        var firstInputPts = Long.MIN_VALUE
+        var firstOutputPts = Long.MIN_VALUE
+        var guard = 0
+        try {
+            while (!outputDone && guard < 100_000) {
+                guard += 1
+                if (!inputDone) {
+                    val index = codec.dequeueInputBuffer(10_000)
+                    if (index >= 0) {
+                        val buffer = codec.getInputBuffer(index)!!
+                        buffer.clear()
+                        val room = buffer.capacity() / 2
+                        val take = minOf(room, pcm.size - offset)
+                        if (take <= 0) {
+                            codec.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
+                            inputDone = true
+                        } else {
+                            val pts = (offset / channels).toLong() * 1_000_000L / rate
+                            if (firstInputPts == Long.MIN_VALUE) firstInputPts = pts
+                            buffer.order(ByteOrder.nativeOrder()).asShortBuffer().put(pcm, offset, take)
+                            codec.queueInputBuffer(index, 0, take * 2, pts, 0)
+                            offset += take
+                        }
+                    }
+                }
+                val index = codec.dequeueOutputBuffer(info, 10_000)
+                if (index >= 0) {
+                    val buffer = codec.getOutputBuffer(index)!!
+                    buffer.position(info.offset)
+                    buffer.limit(info.offset + info.size)
+                    if ((info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
+                        val copy = ByteBuffer.allocate(info.size)
+                        copy.put(buffer); copy.flip(); csd = copy
+                    } else if (info.size > 0) {
+                        if (firstOutputPts == Long.MIN_VALUE) firstOutputPts = info.presentationTimeUs
+                        val bytes = ByteArray(info.size)
+                        buffer.get(bytes)
+                        units.add(bytes)
+                    }
+                    codec.releaseOutputBuffer(index, false)
+                    if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) outputDone = true
+                }
+            }
+        } finally {
+            runCatching { codec.stop() }
+            ledgerEntry.releaseCalledNs = System.nanoTime()
+            runCatching { codec.release() }
+                .onFailure { ledgerEntry.failure = "release: " + it.javaClass.simpleName }
+            ledgerEntry.releasedNs = System.nanoTime()
+        }
+        val shift =
+            if (firstInputPts != Long.MIN_VALUE && firstOutputPts != Long.MIN_VALUE &&
+                firstOutputPts > firstInputPts
+            ) ((firstOutputPts - firstInputPts) * rate / 1_000_000L).toInt()
+            else 0
+        return Encoded(units, csd ?: ByteBuffer.allocate(0), shift)
+    }
+
+    private fun decode(units: List<ByteArray>, csd: ByteBuffer, rate: Int, channels: Int): ShortArray {
+        val format = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, rate, channels).apply {
+            setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC)
+            if (csd.remaining() > 0) setByteBuffer("csd-0", csd.duplicate())
+        }
+        val ledgerEntry = CodecLedger.open("priming.decoder")
+        val codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
+        codec.configure(format, null, null, 0)
+        codec.start()
+        val out = ArrayList<Short>()
+        val info = MediaCodec.BufferInfo()
+        var next = 0
+        var inputDone = false
+        var outputDone = false
+        var guard = 0
+        try {
+            while (!outputDone && guard < 100_000) {
+                guard += 1
+                if (!inputDone) {
+                    val index = codec.dequeueInputBuffer(10_000)
+                    if (index >= 0) {
+                        if (next >= units.size) {
+                            codec.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
+                            inputDone = true
+                        } else {
+                            val unit = units[next]
+                            val buffer = codec.getInputBuffer(index)!!
+                            buffer.clear(); buffer.put(unit)
+                            codec.queueInputBuffer(
+                                index, 0, unit.size,
+                                next.toLong() * 1024L * 1_000_000L / rate, 0,
+                            )
+                            next += 1
+                        }
+                    }
+                }
+                val index = codec.dequeueOutputBuffer(info, 10_000)
+                if (index >= 0) {
+                    if (info.size > 0) {
+                        val buffer = codec.getOutputBuffer(index)!!
+                        buffer.position(info.offset)
+                        buffer.limit(info.offset + info.size)
+                        val shorts = buffer.order(ByteOrder.nativeOrder()).asShortBuffer()
+                        while (shorts.hasRemaining()) out.add(shorts.get())
+                    }
+                    codec.releaseOutputBuffer(index, false)
+                    if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) outputDone = true
+                }
+            }
+        } finally {
+            runCatching { codec.stop() }
+            ledgerEntry.releaseCalledNs = System.nanoTime()
+            runCatching { codec.release() }
+                .onFailure { ledgerEntry.failure = "release: " + it.javaClass.simpleName }
+            ledgerEntry.releasedNs = System.nanoTime()
+        }
+        return ShortArray(out.size) { out[it] }
+    }
+
+    /** Report the measurement for a configuration and burst position. */
+    fun probe(rate: Int, channels: Int, bitrate: Int, at: Int): String {
+        val out = JSONObject().put("rate", rate).put("channels", channels)
+            .put("bitrate", bitrate).put("at", at)
+        return try {
+            val r = measure(rate, channels, bitrate, at, useCache = false)
+            out.put("ok", true).put("delayFrames", r.delayFrames).put("method", r.method)
+                .put("observedOnset", r.observedOnset).put("expectedOnset", r.expectedOnset)
+                .put("delayUs", r.delayFrames.toLong() * 1_000_000L / rate).toString()
+        } catch (e: Exception) {
+            out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message).toString()
+        }
+    }
+}