@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/PrimingTrim.kt

@@ -0,0 +1,337 @@
+package dev.pictor.video
+
+import android.content.Context
+import android.media.MediaCodec
+import android.media.MediaExtractor
+import android.media.MediaFormat
+import android.media.MediaMuxer
+import java.io.File
+import java.nio.ByteBuffer
+import org.json.JSONObject
+
+/**
+ * Removing an encoder's priming, in one place, for everything that publishes
+ * audio.
+ *
+ * ## Why it is shared
+ *
+ * An AAC encoder emits priming samples before any content: they are delay, not
+ * sound. The export already removed them at the final mux, and preview did not
+ * — so a preview played about 43 ms behind its picture while an export of the
+ * same project was aligned. Two paths publishing audio, one of them correcting.
+ *
+ * Rather than teach preview the same trick, the trick moved here. There is now
+ * one definition of "remove the priming", and a third caller cannot be added
+ * without using it.
+ *
+ * ## Every unit is kept, and the start moves instead
+ *
+ * Dropping the priming access units was the obvious correction and it destroys
+ * content. Measured, on one deterministic source with a marker at sample zero:
+ *
+ * | | frames | peak over first 60 ms | markers |
+ * |---|---|---|---|
+ * | full decode | 242688 | 0.309 | 42, 1042, 2542, 4942 |
+ * | decode minus 2045 frames, in memory | 240643 | 0.309 | 0, 1000, 2500, 4900 |
+ * | two access units dropped | 240640 | **0** | 999, 2499, 4899 |
+ * | timestamps shifted, units kept | 240643 | 0.309 | 0, 1000, 2500, 4900 |
+ *
+ * The unit-dropping result differs from the ideal by three samples and loses
+ * the marker completely, so it was never about the samples: an AAC decoder
+ * reconstructs each frame from an overlapping transform of its predecessor, and
+ * a unit that never reaches it takes the first real frame down with it.
+ *
+ * So nothing is dropped. Every unit is written and the first ones present
+ * *before* zero, which is how AAC priming is conventionally signalled. The
+ * shift is sample-exact — there is no grid to round to when nothing is being
+ * removed, which also retires the residual the old approach had to report.
+ *
+ * `MediaMuxer` turns the negative start into container metadata: the written
+ * file reports `encoder-delay: 2045`, and `MediaExtractor` reads it back. A
+ * decoder that starts at zero has already seen the priming and has its context;
+ * one that honours the delay skips exactly it and nothing else.
+ *
+ * ## Measured, never assumed
+ *
+ * The frame count comes from [AacPriming.measure] for the encoder configuration
+ * actually in the file. Hard-coding 2048 — or 43 ms — would be right for one
+ * encoder on one device and quietly wrong elsewhere.
+ *
+ * ## Payload bytes are carried, not re-encoded
+ *
+ * Every retained access unit is copied byte for byte. Only its timestamp
+ * changes, and only by the shift the dropped units account for.
+ */
+internal object PrimingTrim {
+
+    /** AAC frames per access unit. Not configurable; it is what AAC is. */
+    private const val FRAME_SIZE = 1024
+
+    /** What a trim did, for reporting and for assertions. */
+    class Plan(
+        val primingFrames: Int,
+        val unitsDropped: Int,
+        val residualFrames: Int,
+        val shiftUs: Long,
+    ) {
+        fun into(out: JSONObject): JSONObject =
+            out.put("primingFrames", primingFrames)
+                .put("primingUnitsDropped", unitsDropped)
+                .put("primingResidualFrames", residualFrames)
+                .put("primingShiftUs", shiftUs)
+    }
+
+    /**
+     * How far to move the start, given a measured delay and the track's rate.
+     *
+     * Sample-exact. `unitsDropped` is always zero and stays in the report so a
+     * reader can see that nothing was discarded — it is the field that used to
+     * be two.
+     */
+    fun plan(primingFrames: Int, sampleRate: Int): Plan {
+        val rate = if (sampleRate > 0) sampleRate else AudioMixer.SAMPLE_RATE
+        return Plan(
+            primingFrames = primingFrames,
+            unitsDropped = 0,
+            residualFrames = 0,
+            shiftUs = primingFrames.toLong() * 1_000_000L / rate,
+        )
+    }
+
+    /**
+     * The delay this file's encoder configuration produces.
+     *
+     * Read from the track rather than passed in, so a caller cannot supply a
+     * confident number about a different file.
+     */
+    fun measureFor(context: Context, uri: String): Int {
+        val extractor = MediaExtractor()
+        try {
+            if (uri.startsWith("/")) extractor.setDataSource(uri)
+            else extractor.setDataSource(context, android.net.Uri.parse(uri), null)
+            for (i in 0 until extractor.trackCount) {
+                val f = extractor.getTrackFormat(i)
+                if (!(f.getString(MediaFormat.KEY_MIME) ?: "").startsWith("audio/")) continue
+                val rate = f.getInteger(MediaFormat.KEY_SAMPLE_RATE)
+                val channels = f.getInteger(MediaFormat.KEY_CHANNEL_COUNT)
+                val bitrate =
+                    if (f.containsKey(MediaFormat.KEY_BIT_RATE)) f.getInteger(MediaFormat.KEY_BIT_RATE)
+                    else 128_000
+                return AacPriming.measure(rate, channels, bitrate).delayFrames
+            }
+        } catch (e: Exception) {
+            // A file that cannot be opened has no priming to remove; the caller
+            // will fail on it for its own reasons.
+        } finally {
+            runCatching { extractor.release() }
+        }
+        return 0
+    }
+
+    /**
+     * Copy one track's samples, dropping priming units and rebasing the rest.
+     *
+     * Returns the number written, or `-1` if it stopped because it was asked to
+     * — a negative is not a count, it is the reason.
+     *
+     * `limitUs` drops samples starting at or after the compiled end: an encoder
+     * emits whole frames, so the last one runs past the timeline and would make
+     * the container longer than the project.
+     */
+    fun copyTrack(
+        extractor: MediaExtractor,
+        muxer: MediaMuxer,
+        track: Int,
+        limitUs: Long,
+        plan: Plan,
+        beatEpoch: Long,
+        keepGoing: () -> Boolean,
+        beatTag: String = "muxBatch",
+    ): Int {
+        val buffer = ByteBuffer.allocate(4 * 1024 * 1024)
+        val info = MediaCodec.BufferInfo()
+        var count = 0
+        var skipped = 0
+        while (true) {
+            if (!keepGoing()) return -1
+            buffer.clear()
+            val size = extractor.readSampleData(buffer, 0)
+            if (size < 0) break
+            /*
+              Timestamps are carried across unchanged until the rebase below.
+
+              Clamping a negative one to zero would move priming samples into the
+              audible timeline and shift everything after them; the muxer is
+              given exactly what the source said, less the shift.
+            */
+            var pts = extractor.sampleTime
+            if (skipped < plan.unitsDropped) {
+                // Priming units are discarded whole; nothing decodable is lost
+                // because nothing in them is content.
+                skipped += 1
+                if (!extractor.advance()) break
+                continue
+            }
+            /*
+              Rebased, then limited.
+
+              The tail limit is a statement about the finished timeline, so it
+              has to be applied to the timestamps the file will actually carry —
+              comparing against the pre-shift value would trim a different amount
+              of audio than the caller asked for.
+            */
+            pts -= plan.shiftUs
+            if (limitUs > 0 && pts >= limitUs) break
+            info.offset = 0
+            info.size = size
+            info.presentationTimeUs = pts
+            info.flags = extractor.sampleFlags
+            muxer.writeSampleData(track, buffer, info)
+            count += 1
+            // A batch rather than every sample: enough to prove the copy is
+            // alive without making the beat itself the work.
+            if (count % 8 == 0) ExportHeartbeat.beat(beatTag, beatEpoch)
+            if (!extractor.advance()) break
+        }
+        return count
+    }
+
+    /**
+     * Rewrite an audio-only file keeping every access unit, and move the
+     * presentation start instead.
+     *
+     * ## Why the units have to stay
+     *
+     * Measured: dropping the two priming units removes the marker at timeline
+     * zero completely — decoded peak over the first 60 ms falls to zero —
+     * while removing exactly the same delay from a full decode, in memory,
+     * preserves it. The two results differ by three samples, so it is not the
+     * samples. An AAC decoder reconstructs each frame from an overlapping
+     * transform of its predecessor, so a unit that never reaches it takes the
+     * first real frame down with it.
+     *
+     * So every unit is written, and the first ones simply present *before* zero.
+     * A decoder that starts at zero has already seen them and has its overlap
+     * context; a decoder that honours the negative span skips exactly the delay
+     * and nothing else.
+     *
+     * Payload bytes are copied unchanged, as in [copyTrack]. Only timestamps
+     * move, and only by the measured delay.
+     */
+    fun shiftAudioFile(
+        context: Context,
+        source: File,
+        into: File,
+        /**
+         * The compiled end, or 0 to keep everything.
+         *
+         * The same tail policy the export mux applies: a unit starting at or
+         * after the timeline's end is entirely past the last frame and is
+         * dropped, while one that merely *extends* past the end is kept. An
+         * encoder emits whole frames, so the alternative — cutting at the exact
+         * sample — would need a re-encode, and the leftover is at most one
+         * frame of padding rather than the 643 frames a missing limit leaves.
+         */
+        limitUs: Long = 0,
+        keepGoing: () -> Boolean = { true },
+    ): JSONObject {
+        val out = JSONObject()
+        val extractor = MediaExtractor()
+        var muxer: MediaMuxer? = null
+        try {
+            extractor.setDataSource(source.absolutePath)
+            var track = -1
+            for (i in 0 until extractor.trackCount) {
+                val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
+                if (mime.startsWith("audio/")) { track = i; break }
+            }
+            if (track < 0) return out.put("ok", false).put("error", "no audio track")
+
+            val format = extractor.getTrackFormat(track)
+            val rate =
+                if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE))
+                    format.getInteger(MediaFormat.KEY_SAMPLE_RATE)
+                else AudioMixer.SAMPLE_RATE
+            val primingFrames = measureFor(context, source.absolutePath)
+            // Sample-exact, not unit-quantised: nothing is being dropped, so
+            // there is no grid to round to.
+            val shiftUs = primingFrames.toLong() * 1_000_000L / rate
+
+            extractor.selectTrack(track)
+            muxer = MediaMuxer(into.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
+            val outTrack = muxer.addTrack(format)
+            muxer.start()
+
+            val buffer = ByteBuffer.allocate(4 * 1024 * 1024)
+            val info = MediaCodec.BufferInfo()
+            var count = 0
+            var negatives = 0
+            var dropped = 0
+            var firstPts = Long.MIN_VALUE
+            while (true) {
+                if (!keepGoing()) return out.put("ok", false).put("cancelled", true)
+                buffer.clear()
+                val size = extractor.readSampleData(buffer, 0)
+                if (size < 0) break
+                val pts = extractor.sampleTime - shiftUs
+                // Limited on the shifted timestamp, because that is what the
+                // file will carry; comparing the pre-shift value would trim a
+                // different amount than the caller asked for.
+                if (limitUs > 0 && pts >= limitUs) { dropped += 1; break }
+                if (firstPts == Long.MIN_VALUE) firstPts = pts
+                if (pts < 0) negatives += 1
+                info.offset = 0
+                info.size = size
+                info.presentationTimeUs = pts
+                info.flags = extractor.sampleFlags
+                muxer.writeSampleData(track, buffer, info)
+                count += 1
+                if (!extractor.advance()) break
+            }
+
+            return out.put("ok", true)
+                .put("samples", count)
+                .put("primingFrames", primingFrames)
+                .put("shiftUs", shiftUs)
+                .put("negativePtsUnits", negatives)
+                .put("firstPtsUs", firstPts)
+                .put("tailUnitsDropped", dropped)
+                .put("limitUs", limitUs)
+        } catch (e: Exception) {
+            return out.put("ok", false).put("error", e.javaClass.simpleName + ": " + e.message)
+        } finally {
+            runCatching { muxer?.stop() }
+            runCatching { muxer?.release() }
+            runCatching { extractor.release() }
+        }
+    }
+
+    /** What a written container actually says about its start. */
+    fun inspect(file: File): JSONObject {
+        val out = JSONObject()
+        val extractor = MediaExtractor()
+        try {
+            extractor.setDataSource(file.absolutePath)
+            for (i in 0 until extractor.trackCount) {
+                val f = extractor.getTrackFormat(i)
+                if (!(f.getString(MediaFormat.KEY_MIME) ?: "").startsWith("audio/")) continue
+                out.put("mime", f.getString(MediaFormat.KEY_MIME))
+                if (f.containsKey(MediaFormat.KEY_DURATION)) {
+                    out.put("durationUs", f.getLong(MediaFormat.KEY_DURATION))
+                }
+                for (key in listOf("encoder-delay", "encoder-padding", "exo-pts-offset-us")) {
+                    if (f.containsKey(key)) out.put(key, f.getInteger(key))
+                }
+                extractor.selectTrack(i)
+                out.put("firstSampleTimeUs", extractor.sampleTime)
+                break
+            }
+        } catch (e: Exception) {
+            out.put("error", e.javaClass.simpleName)
+        } finally {
+            runCatching { extractor.release() }
+        }
+        return out
+    }
+
+}

package/android/src/main/java/dev/pictor/video/RepeatingEncoderFactory.kt

@@ -0,0 +1,97 @@
+package dev.pictor.video
+
+import android.content.Context
+import android.media.MediaCodecInfo
+import android.media.MediaCodecList
+import android.media.MediaFormat
+import android.media.metrics.LogSessionId
+import android.os.Build
+import androidx.annotation.RequiresApi
+import androidx.media3.common.Format
+import androidx.media3.common.MimeTypes
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.transformer.Codec
+import androidx.media3.transformer.DefaultCodec
+import androidx.media3.transformer.DefaultEncoderFactory
+import androidx.media3.transformer.ExportException
+
+/**
+ * An encoder told to repeat its last frame if nothing new arrives.
+ *
+ * `KEY_REPEAT_PREVIOUS_FRAME_AFTER` is the only frame-*repetition* mechanism
+ * Media3 can reach without a custom shader program, and slow motion needs
+ * repetition: `FrameDropEffect` can thin a stream toward a target rate but
+ * cannot thicken one. This exists to find out whether that mechanism is a
+ * production answer or only a plausible-looking one.
+ *
+ * **It is expected to fail a specific test, and the test was chosen first.** The
+ * encoder sits *downstream* of the GL pipeline, so a frame it repeats was never
+ * drawn by the scene interpreter. If the repeated frames do not pass through the
+ * interpreter then overlays, transforms and opacity cannot be applied to them,
+ * and the mechanism is rejected however correct the timestamps look. Measuring
+ * scene executions against output frames is what tells those apart.
+ *
+ * Gate-only. Nothing here is reachable from a public exporter.
+ */
+@UnstableApi
+@RequiresApi(Build.VERSION_CODES.S)
+internal class RepeatingEncoderFactory(
+    private val context: Context,
+    /** Repeat the previous frame after this long with no new one. */
+    private val repeatAfterUs: Long,
+    private val targetFps: Int,
+) : Codec.EncoderFactory {
+
+    private val fallback = DefaultEncoderFactory.Builder(context).build()
+
+    override fun createForAudioEncoding(format: Format, logSessionId: LogSessionId?): Codec =
+        fallback.createForAudioEncoding(format, logSessionId)
+
+    override fun createForVideoEncoding(format: Format, logSessionId: LogSessionId?): Codec {
+        val mime = MimeTypes.VIDEO_H264
+        val name = encoderFor(mime)
+            ?: throw ExportException.createForCodec(
+                IllegalStateException("no encoder for $mime"),
+                ExportException.ERROR_CODE_ENCODING_FAILED,
+                ExportException.CodecInfo(
+                    /* configurationFormat= */ format.toString(),
+                    /* isVideo= */ true,
+                    /* isDecoder= */ false,
+                    /* name= */ null,
+                ),
+            )
+
+        val mediaFormat = MediaFormat.createVideoFormat(mime, format.width, format.height)
+        mediaFormat.setInteger(
+            MediaFormat.KEY_COLOR_FORMAT,
+            MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface,
+        )
+        // A bitrate the encoder will accept for this size; the gate compares
+        // cadence, not quality, and a missing bitrate is a configure failure.
+        val bitrate = if (format.averageBitrate != Format.NO_VALUE) format.averageBitrate else 4_000_000
+        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate)
+        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, targetFps)
+        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
+        if (repeatAfterUs > 0) {
+            mediaFormat.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, repeatAfterUs)
+        }
+
+        return DefaultCodec(
+            context,
+            format,
+            mediaFormat,
+            name,
+            /* isDecoder= */ false,
+            /* outputSurface= */ null,
+        )
+    }
+
+    private fun encoderFor(mime: String): String? {
+        val list = MediaCodecList(MediaCodecList.REGULAR_CODECS)
+        for (info in list.codecInfos) {
+            if (!info.isEncoder) continue
+            if (info.supportedTypes.any { it.equals(mime, ignoreCase = true) }) return info.name
+        }
+        return null
+    }
+}

package/android/src/main/java/dev/pictor/video/SceneProgram.kt

@@ -0,0 +1,169 @@
+package dev.pictor.video
+
+import org.json.JSONArray
+import org.json.JSONObject
+
+/**
+ * A resolved scene, validated, in the form the GL interpreter executes.
+ *
+ * **Native never receives a project.** It receives a scene that
+ * `resolveVideoScene` already produced — ids, indices and numbers — and its only
+ * job is to refuse anything it cannot faithfully execute and then execute the
+ * rest. There is no timeline arithmetic here, no window mapping, no
+ * placement decision: those exist once, in TypeScript, and this would be the
+ * second copy if it had them.
+ *
+ * ## Refuse, never skip
+ *
+ * Every unknown command, unsupported version or malformed field fails the whole
+ * run before Transformer starts. A skipped command is a frame that is quietly
+ * missing something the timeline said was there, encoded permanently — and
+ * unlike preview, nothing downstream will ever notice.
+ */
+internal class SceneProgram private constructor(
+    val timeUs: Long,
+    val projectWidth: Int,
+    val projectHeight: Int,
+    val clearColor: Int,
+    val clipId: String,
+    val windowIndex: Int,
+) {
+    /** What went wrong, or null if nothing did. */
+    class Invalid(val reason: String)
+
+    companion object {
+        /** The only schema this build executes. */
+        const val SUPPORTED_VERSION = 2
+
+        /** Commands this spike implements. Anything else is refused, not skipped. */
+        private val SUPPORTED_OPS = setOf("clear", "video")
+
+        /**
+         * Parse and validate, or say why not.
+         *
+         * Everything is checked *before* Transformer is started: a run that
+         * fails halfway has already written a partial file and taken an encoder
+         * with it, and none of these failures need to get that far.
+         */
+        fun parse(json: String): Pair<SceneProgram?, Invalid?> {
+            val root = try {
+                JSONObject(json)
+            } catch (e: Exception) {
+                return null to Invalid("malformed JSON: ${e.message}")
+            }
+
+            val version = root.optInt("version", -1)
+            if (version != SUPPORTED_VERSION) {
+                return null to Invalid("unsupported scene version $version, expected $SUPPORTED_VERSION")
+            }
+
+            if (!root.has("timeUs") || !root.has("projectSize") || !root.has("commands")) {
+                return null to Invalid("scene is missing timeUs, projectSize or commands")
+            }
+            val timeUs = root.optLong("timeUs", -1)
+            if (timeUs < 0) return null to Invalid("timeUs is not a non-negative integer")
+
+            val size = root.optJSONObject("projectSize")
+                ?: return null to Invalid("projectSize is not an object")
+            val width = size.optDouble("width", Double.NaN)
+            val height = size.optDouble("height", Double.NaN)
+            if (!finite(width) || !finite(height) || width <= 0 || height <= 0) {
+                return null to Invalid("projectSize is not a positive finite size")
+            }
+
+            val commands: JSONArray = root.optJSONArray("commands")
+                ?: return null to Invalid("commands is not an array")
+            if (commands.length() == 0) return null to Invalid("commands is empty")
+
+            var clearColor: Int? = null
+            var clipId: String? = null
+            var windowIndex = -1
+
+            for (i in 0 until commands.length()) {
+                val command = commands.optJSONObject(i)
+                    ?: return null to Invalid("command $i is not an object")
+                val op = command.optString("op", "")
+                if (op !in SUPPORTED_OPS) {
+                    // Includes `overlay`, which this spike does not implement.
+                    // Refusing is the point: a silently dropped overlay is a
+                    // wrong frame nothing later can detect.
+                    return null to Invalid("command $i has unsupported op '$op'")
+                }
+                when (op) {
+                    "clear" -> {
+                        // Order is part of the representation, not a convention:
+                        // the resolver puts `clear` first and an interpreter that
+                        // accepted it elsewhere would be deciding paint order.
+                        if (i != 0) return null to Invalid("clear must be the first command, found at $i")
+                        val color = parseColor(command.optString("color", ""))
+                            ?: return null to Invalid("clear has an unparseable color")
+                        clearColor = color
+                    }
+                    "video" -> {
+                        if (clearColor == null) return null to Invalid("video command before any clear")
+                        if (clipId != null) return null to Invalid("more than one video command")
+                        val id = command.optString("clipId", "")
+                        if (id.isEmpty()) return null to Invalid("video command has no clipId")
+                        val window = command.optInt("windowIndex", -1)
+                        if (window < 0) return null to Invalid("video command has no windowIndex")
+                        // This gate exports a single source, so window 0 is the
+                        // only identity the media can satisfy. A scene naming
+                        // another one is describing a timeline this run does not
+                        // have, and running it would encode the right pixels
+                        // under the wrong claim.
+                        if (window != 0) {
+                            return null to Invalid("windowIndex $window has no window in a single-source run")
+                        }
+                        val matrix = command.optJSONArray("matrix")
+                            ?: return null to Invalid("video command has no matrix")
+                        if (matrix.length() != 6) return null to Invalid("matrix is not six numbers")
+                        for (k in 0 until 6) {
+                            if (!finite(matrix.optDouble(k, Double.NaN))) {
+                                return null to Invalid("matrix[$k] is not finite")
+                            }
+                        }
+                        // This spike executes identity video only. A transform is
+                        // a real command it cannot honour, so it is refused rather
+                        // than approximated.
+                        if (!isIdentity(matrix)) {
+                            return null to Invalid("only the identity matrix is supported in this spike")
+                        }
+                        val opacity = command.optDouble("opacity", Double.NaN)
+                        if (!finite(opacity)) return null to Invalid("opacity is not finite")
+                        if (opacity != 1.0) {
+                            return null to Invalid("only full opacity is supported in this spike")
+                        }
+                        clipId = id
+                        windowIndex = window
+                    }
+                }
+            }
+
+            if (clearColor == null) return null to Invalid("no clear command")
+            if (clipId == null) return null to Invalid("no video command")
+
+            return SceneProgram(
+                timeUs = timeUs,
+                projectWidth = width.toInt(),
+                projectHeight = height.toInt(),
+                clearColor = clearColor,
+                clipId = clipId,
+                windowIndex = windowIndex,
+            ) to null
+        }
+
+        private fun finite(value: Double) = !value.isNaN() && !value.isInfinite()
+
+        private fun isIdentity(m: JSONArray): Boolean =
+            m.optDouble(0) == 1.0 && m.optDouble(1) == 0.0 && m.optDouble(2) == 0.0 &&
+                m.optDouble(3) == 1.0 && m.optDouble(4) == 0.0 && m.optDouble(5) == 0.0
+
+        /** `#aarrggbb` or `#rrggbb` to packed ARGB. */
+        private fun parseColor(hex: String): Int? {
+            val raw = hex.removePrefix("#")
+            if (raw.length != 6 && raw.length != 8) return null
+            val value = raw.toLongOrNull(16) ?: return null
+            return if (raw.length == 8) value.toInt() else (0xff000000L or value).toInt()
+        }
+    }
+}