@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/AudioProbe.kt

@@ -0,0 +1,162 @@
+package dev.pictor.video
+
+import android.content.Context
+import android.media.MediaExtractor
+import android.media.MediaFormat
+import android.media.MediaMetadataRetriever
+import android.net.Uri
+import org.json.JSONObject
+
+/**
+ * What a file actually contains, asked before it is allowed into a project.
+ *
+ * ## Why the host probes and the mixer does not
+ *
+ * `compileAudioMixProgram` runs in plain Node with no decoder, so it can only
+ * believe what the document tells it: `metadata.hasAudio === false` means
+ * silence and anything else means audio. A project that claimed audio a file
+ * did not have already cost one whole export, which failed at the encoding
+ * stage with nothing on screen to explain it — the encoder had been asked to
+ * decode a track that was not there.
+ *
+ * So this is the only thing that gets to answer the question, and it answers it
+ * by opening the file rather than by reading its name. A `.m4a` carrying only
+ * cover art is a real thing, and it is indistinguishable from music until
+ * someone looks at the tracks.
+ *
+ * ## Everything is an outcome
+ *
+ * A probe of an unreadable file is a normal result, not an exception: the user
+ * picked something, and "it could not be read" is the answer. Every failure
+ * comes back as `ok: false` with a reason, and nothing here throws into JS.
+ *
+ * Nothing in the returned JSON identifies the file. The display name is the
+ * host's to carry — a `content://` URI embeds an id and sometimes a user's own
+ * folder names, and none of that belongs in a log.
+ */
+object AudioProbe {
+
+    /**
+     * Open [uri], report its audio.
+     *
+     * Returns JSON: `{ok, durationUs, hasAudioTrack, seekable, mimeType}` or
+     * `{ok: false, reason}`.
+     */
+    fun probe(context: Context, uri: String): JSONObject {
+        val out = JSONObject()
+        val parsed =
+            try {
+                Uri.parse(uri)
+            } catch (e: Exception) {
+                return out.put("ok", false).put("reason", "unparseable uri")
+            }
+
+        val extractor = MediaExtractor()
+        try {
+            extractor.setDataSource(context, parsed, null)
+        } catch (e: Exception) {
+            // The common real case: a picker URI whose permission has lapsed,
+            // or a container no extractor on this device can open.
+            extractor.release()
+            return out.put("ok", false).put("reason", reasonOf(e))
+        }
+
+        try {
+            var audioTrack = -1
+            var mimeType = ""
+            for (i in 0 until extractor.trackCount) {
+                val format = extractor.getTrackFormat(i)
+                val mime = format.getString(MediaFormat.KEY_MIME) ?: continue
+                if (mime.startsWith("audio/")) {
+                    audioTrack = i
+                    mimeType = mime
+                    break
+                }
+            }
+
+            val durationUs = durationOf(context, parsed, extractor, audioTrack)
+
+            /*
+                Seekability, tested rather than assumed.
+
+                Trimming, fades and preview scrubbing are all seeks, so a source
+                that cannot seek would accept every edit Music offers and honour
+                none of them. There is no flag for this on `MediaExtractor`, so
+                the question is put to the file: seek to the middle and see
+                whether the next sample actually moved. A container that ignores
+                the request answers 0 and is refused.
+            */
+            var seekable = false
+            if (audioTrack >= 0 && durationUs > 0) {
+                seekable =
+                    try {
+                        extractor.selectTrack(audioTrack)
+                        val target = durationUs / 2
+                        extractor.seekTo(target, MediaExtractor.SEEK_TO_CLOSEST_SYNC)
+                        val landed = extractor.sampleTime
+                        // A quarter of the file is a wide margin on purpose: a
+                        // sparse sync table can land well short of the target
+                        // and still be perfectly seekable.
+                        landed > 0 && landed > durationUs / 4
+                    } catch (e: Exception) {
+                        false
+                    }
+            }
+
+            return out
+                .put("ok", true)
+                .put("durationUs", durationUs)
+                .put("hasAudioTrack", audioTrack >= 0)
+                .put("seekable", seekable)
+                .put("mimeType", mimeType)
+        } catch (e: Exception) {
+            return out.put("ok", false).put("reason", reasonOf(e))
+        } finally {
+            extractor.release()
+        }
+    }
+
+    /**
+     * The track's own duration, falling back to the container's.
+     *
+     * A track format usually carries `KEY_DURATION`, and when it does it is the
+     * more precise answer. Some containers leave it off the track and put it
+     * only in the metadata, so both are asked before the file is called
+     * unreadable.
+     */
+    private fun durationOf(
+        context: Context,
+        uri: Uri,
+        extractor: MediaExtractor,
+        audioTrack: Int,
+    ): Long {
+        if (audioTrack >= 0) {
+            val format = extractor.getTrackFormat(audioTrack)
+            if (format.containsKey(MediaFormat.KEY_DURATION)) {
+                val fromTrack = format.getLong(MediaFormat.KEY_DURATION)
+                if (fromTrack > 0) return fromTrack
+            }
+        }
+        val retriever = MediaMetadataRetriever()
+        return try {
+            retriever.setDataSource(context, uri)
+            val ms =
+                retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLongOrNull()
+            // Milliseconds there, microseconds everywhere in Pictor. The wrong
+            // one makes every song a thousand times too short.
+            if (ms != null && ms > 0) ms * 1000 else 0
+        } catch (e: Exception) {
+            0
+        } finally {
+            try {
+                retriever.release()
+            } catch (e: Exception) {
+                // Releasing a retriever that never opened is not a failure
+                // worth reporting over the one that brought us here.
+            }
+        }
+    }
+
+    private fun reasonOf(e: Exception): String =
+        e.javaClass.simpleName + (e.message?.let { ": $it" } ?: "")
+}

package/android/src/main/java/dev/pictor/video/CadenceShaderProgram.kt

@@ -0,0 +1,337 @@
+package dev.pictor.video
+
+import android.opengl.GLES20
+import androidx.media3.common.GlObjectsProvider
+import androidx.media3.common.GlTextureInfo
+import androidx.media3.common.VideoFrameProcessingException
+import androidx.media3.common.util.GlProgram
+import androidx.media3.common.util.GlUtil
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.effect.GlEffect
+import androidx.media3.effect.GlShaderProgram
+import java.util.ArrayDeque
+import java.util.concurrent.Executor
+import java.util.concurrent.atomic.AtomicLong
+
+/**
+ * Puts frames on the target grid, **upstream of the scene interpreter**.
+ *
+ * `setSpeed` re-times without resampling: the source frame count survives every
+ * rate and only the interval moves. `FrameDropEffect` closes the rates above 1×,
+ * where the fix is to remove frames. Below 1× the fix is to add them, and
+ * Media3 has no public mechanism that does — the encoder's repeat interval is
+ * wall-clock driven, never fires in a non-real-time transcode, and sits
+ * downstream of the GL pipeline where a repeated frame would never be drawn by
+ * the interpreter at all.
+ *
+ * So repetition happens here, before the interpreter, and every emitted frame
+ * carries its own output presentation timestamp into scene selection. A repeat
+ * is a real frame with a real time: overlays, transforms and opacity apply to it
+ * exactly as they do to any other.
+ *
+ * ## It computes nothing
+ *
+ * The emission plan is compiled by `resampleClip` in TypeScript and arrives as a
+ * count per input frame plus the timestamps to stamp them with. This class walks
+ * that list. It holds no rate, no duration and no grid arithmetic — the whole of
+ * its cleverness is the ownership protocol below.
+ *
+ * ## Ownership
+ *
+ * Media3 lends an input texture and takes it back when told. An output frame
+ * handed downstream may be read long after, so an input must not be released
+ * while any output still refers to it — but outputs here are **copies**, made
+ * while the input is still owned, which is what makes repetition safe at all.
+ *
+ * The input is released only after its last copy has been made. Copies are
+ * emitted one at a time, bounded by the texture pool, and the next is made when
+ * the downstream returns one. That bound is the backpressure: a slow consumer
+ * stalls this program instead of growing its buffer.
+ */
+@UnstableApi
+internal class CadenceEffect(
+    /** Copies each input frame owes, in arrival order. Compiled in TypeScript. */
+    private val emitsPerInput: IntArray,
+    /** Output timestamps, flattened in emission order. Compiled in TypeScript. */
+    private val outputPtsUs: LongArray,
+    /** Outstanding output frames allowed at once. */
+    private val maxPending: Int = 2,
+    private val failAfterEmits: Int = -1,
+    /** Gate-only: go quiet after N frames — no emit, no beat, no error. */
+    private val stallAfterEmits: Int = -1,
+) : GlEffect {
+
+    val emitted = AtomicLong(0)
+    val inputsSeen = AtomicLong(0)
+    val glCreated = AtomicLong(0)
+    val glDeleted = AtomicLong(0)
+    val peakPending = AtomicLong(0)
+    val stalls = AtomicLong(0)
+    private val problems = mutableListOf<String>()
+
+    fun trace(): String = synchronized(problems) {
+        "inputs=" + inputsSeen.get() + " emitted=" + emitted.get() +
+            " glCreated=" + glCreated.get() + " glDeleted=" + glDeleted.get() +
+            " peakPending=" + peakPending.get() + " stalls=" + stalls.get() +
+            (if (problems.isEmpty()) "" else " problems=" + problems.joinToString("|"))
+    }
+
+    private fun problem(message: String) {
+        synchronized(problems) { if (problems.size < 8) problems.add(message) }
+    }
+
+    override fun toGlShaderProgram(context: android.content.Context, useHdr: Boolean): GlShaderProgram =
+        Program()
+
+    private inner class Program : GlShaderProgram {
+        private var inputListener: GlShaderProgram.InputListener? = null
+        private var outputListener: GlShaderProgram.OutputListener? = null
+        private var errorExecutor: Executor? = null
+        private var errorListener: GlShaderProgram.ErrorListener? = null
+
+        private var glProgram: GlProgram? = null
+        private var width = 0
+        private var height = 0
+
+        /** Textures this program owns. Created once, reused, freed in [release]. */
+        private val all = mutableListOf<GlTextureInfo>()
+        private val free = ArrayDeque<GlTextureInfo>()
+        private var pending = 0
+
+        /** The borrowed input, held only while copies of it are still owed. */
+        private var heldInput: GlTextureInfo? = null
+        private var owed = 0
+        private var emitCursor = 0
+        private var inputOrdinal = 0
+        private var endOfStreamSignalled = false
+        private var released = false
+        /** Captured once: a beat is only evidence for the job that made this. */
+        private val beatEpoch = ExportHeartbeat.currentEpoch()
+
+        override fun setInputListener(listener: GlShaderProgram.InputListener) {
+            inputListener = listener
+            // Only invite a frame when there is somewhere to put its first copy.
+            if (heldInput == null && !released) listener.onReadyToAcceptInputFrame()
+        }
+
+        override fun setOutputListener(listener: GlShaderProgram.OutputListener) {
+            outputListener = listener
+        }
+
+        override fun setErrorListener(executor: Executor, listener: GlShaderProgram.ErrorListener) {
+            errorExecutor = executor
+            errorListener = listener
+        }
+
+        override fun queueInputFrame(
+            provider: GlObjectsProvider,
+            input: GlTextureInfo,
+            presentationTimeUs: Long,
+        ) {
+            if (released) return
+            inputsSeen.incrementAndGet()
+            try {
+                ensureGl(provider, input.width, input.height)
+
+                val ordinal = inputOrdinal
+                inputOrdinal += 1
+                owed = if (ordinal < emitsPerInput.size) emitsPerInput[ordinal] else 0
+                if (owed == 0) {
+                    /*
+                      A frame the plan does not use.
+
+                      Released immediately and never copied — this is how the
+                      program drops above 1× without a second mechanism. The
+                      interpreter downstream never sees it, so scene executions
+                      stay equal to output frames.
+                    */
+                    inputListener?.onInputFrameProcessed(input)
+                    inputListener?.onReadyToAcceptInputFrame()
+                    return
+                }
+                heldInput = input
+                pump()
+            } catch (e: Exception) {
+                fail(e)
+            }
+        }
+
+        /**
+         * Emit as many copies as the pool allows, then release the input if done.
+         *
+         * The input is released *after* the last copy, never before: a copy is
+         * made from the borrowed texture, so returning it early would leave a
+         * draw reading memory Media3 has taken back.
+         */
+        private fun pump() {
+            val input = heldInput ?: return
+            while (owed > 0 && pending < maxPending) {
+                val target = free.poll() ?: break
+                val pts = if (emitCursor < outputPtsUs.size) outputPtsUs[emitCursor] else 0L
+                emitCursor += 1
+
+                if (stallAfterEmits >= 0 && emitted.get() >= stallAfterEmits) {
+                    // Return holding the input: nothing is emitted, nothing
+                    // beats, and no exception is raised. Silence is the point.
+                    return
+                }
+                if (failAfterEmits >= 0 && emitted.get() >= failAfterEmits) {
+                    throw VideoFrameProcessingException(
+                        RuntimeException("injected cadence failure at emit " + emitted.get()),
+                    )
+                }
+
+                copy(input, target)
+                owed -= 1
+                pending += 1
+                if (pending.toLong() > peakPending.get()) peakPending.set(pending.toLong())
+                emitted.incrementAndGet()
+                // An output frame is the unit the bar is weighted in, so it is
+                // counted here rather than inferred from anything downstream.
+                ExportProgress.frame(beatEpoch)
+                // Liveness, not progress: one repeated copy moves the bar
+                // barely at all and still proves the pipeline is running.
+                ExportHeartbeat.beat("cadence", beatEpoch)
+                outputListener?.onOutputFrameAvailable(target, pts)
+            }
+            if (owed > 0) {
+                // Bounded, so a slow consumer stalls this program rather than
+                // growing an unbounded queue of copies.
+                stalls.incrementAndGet()
+                return
+            }
+            heldInput = null
+            inputListener?.onInputFrameProcessed(input)
+            inputListener?.onReadyToAcceptInputFrame()
+            maybeSignalEnd()
+        }
+
+        private fun copy(input: GlTextureInfo, target: GlTextureInfo) {
+            val program = glProgram ?: throw IllegalStateException("no GL program")
+            GlUtil.focusFramebufferUsingCurrentContext(target.fboId, target.width, target.height)
+            GlUtil.clearFocusedBuffers()
+            program.use()
+            program.setSamplerTexIdUniform("uTexSampler", input.texId, 0)
+            program.bindAttributesAndUniforms()
+            GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
+            GlUtil.checkGlError()
+        }
+
+        private fun ensureGl(provider: GlObjectsProvider, w: Int, h: Int) {
+            if (glProgram != null && w == width && h == height) return
+            releaseGl()
+            width = w
+            height = h
+            val program = GlProgram(VERTEX_SHADER, FRAGMENT_SHADER)
+            program.setBufferAttribute(
+                "aFramePosition",
+                floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f),
+                2,
+            )
+            glProgram = program
+            // The pool is the buffering bound, allocated once and reused. One
+            // spare beyond `maxPending` so a copy can be prepared while the
+            // downstream still holds the previous one.
+            for (i in 0 until maxPending + 1) {
+                val texId = GlUtil.createTexture(w, h, /* useHighPrecisionColorComponents= */ false)
+                val info = provider.createBuffersForTexture(texId, w, h)
+                all.add(info)
+                free.add(info)
+            }
+            glCreated.incrementAndGet()
+        }
+
+        override fun releaseOutputFrame(outputTexture: GlTextureInfo) {
+            if (released) return
+            pending -= 1
+            free.add(outputTexture)
+            try {
+                pump()
+            } catch (e: Exception) {
+                fail(e)
+            }
+        }
+
+        override fun signalEndOfCurrentInputStream() {
+            endOfStreamSignalled = true
+            maybeSignalEnd()
+        }
+
+        /**
+         * End only when nothing is owed.
+         *
+         * Signalling while copies are outstanding would truncate the clip;
+         * emitting anything after the plan is exhausted would extend it past its
+         * duration. The plan's length is the clip's length, so neither happens by
+         * construction — and the guard is here because "by construction" is worth
+         * one comparison.
+         */
+        private fun maybeSignalEnd() {
+            if (endOfStreamSignalled && heldInput == null && owed == 0) {
+                outputListener?.onCurrentOutputStreamEnded()
+            }
+        }
+
+        override fun flush() {
+            // Everything borrowed is dropped and the plan restarts. Media3 owns
+            // the input across a flush, so it is not released here.
+            heldInput = null
+            owed = 0
+            pending = 0
+            emitCursor = 0
+            inputOrdinal = 0
+            endOfStreamSignalled = false
+            free.clear()
+            free.addAll(all)
+            inputListener?.onFlush()
+            inputListener?.onReadyToAcceptInputFrame()
+        }
+
+        override fun release() {
+            released = true
+            if (heldInput != null) problem("released while holding an input")
+            if (pending != 0) problem("released with " + pending + " outputs outstanding")
+            releaseGl()
+        }
+
+        private fun releaseGl() {
+            runCatching { glProgram?.delete() }
+            if (glProgram != null) glDeleted.incrementAndGet()
+            glProgram = null
+            for (info in all) runCatching { info.release() }
+            all.clear()
+            free.clear()
+        }
+
+        private fun fail(e: Exception) {
+            problem(e.javaClass.simpleName + ": " + e.message)
+            val listener = errorListener
+            val executor = errorExecutor
+            val wrapped =
+                if (e is VideoFrameProcessingException) e else VideoFrameProcessingException(e)
+            if (listener != null && executor != null) {
+                executor.execute { listener.onError(wrapped) }
+            }
+        }
+    }
+
+    private companion object {
+        const val VERTEX_SHADER = """
+            attribute vec4 aFramePosition;
+            varying vec2 vTexCoord;
+            void main() {
+              gl_Position = aFramePosition;
+              vTexCoord = aFramePosition.xy * 0.5 + 0.5;
+            }
+        """
+
+        const val FRAGMENT_SHADER = """
+            precision mediump float;
+            uniform sampler2D uTexSampler;
+            varying vec2 vTexCoord;
+            void main() {
+              gl_FragColor = texture2D(uTexSampler, vTexCoord);
+            }
+        """
+    }
+}

package/android/src/main/java/dev/pictor/video/CodecLedger.kt

@@ -0,0 +1,126 @@
+package dev.pictor.video
+
+import java.util.concurrent.atomic.AtomicLong
+import org.json.JSONArray
+import org.json.JSONObject
+
+/**
+ * Every codec mix preparation owns, from creation to the end of release.
+ *
+ * ## Why this exists
+ *
+ * The picture stops advancing after a mix is prepared, and the minimal failing
+ * configuration is preparation alone — no second player, no position pump, no
+ * coordinator. Preparation's distinguishing feature is codecs: four decoders
+ * for the sources, an encoder for the mix, and another encode/decode pair
+ * inside the priming calibration. If one of them outlives the job, the video
+ * decoder is competing for a hardware resource that nothing is going to give
+ * back.
+ *
+ * That is a hypothesis, and this is what turns it into an answer. A `release()`
+ * that was called is not the same as a release that finished, and a `finally`
+ * block that ran is not the same as one that ran without throwing — so both
+ * ends of every codec's life are recorded, with the exception path if there
+ * was one.
+ *
+ * ## What it deliberately does not cover
+ *
+ * The preview video decoder. That belongs to `Media3Player` and lives as long
+ * as the screen does; counting it here would make "live after publication"
+ * always true and the check worthless. Ownership is the point: these are the
+ * codecs a *job* created and must give back before it publishes.
+ */
+internal object CodecLedger {
+
+    private val nextId = AtomicLong(0)
+
+    class Entry(
+        val id: Long,
+        val role: String,
+        val job: String,
+        val createdNs: Long,
+    ) {
+        @Volatile var configuredNs: Long = 0
+        @Volatile var startedNs: Long = 0
+        @Volatile var stoppedNs: Long = 0
+        @Volatile var releaseCalledNs: Long = 0
+        @Volatile var releasedNs: Long = 0
+        @Volatile var failure: String? = null
+
+        val live: Boolean get() = releasedNs == 0L
+
+        fun json(nowNs: Long): JSONObject =
+            JSONObject()
+                .put("id", id)
+                .put("role", role)
+                .put("job", job)
+                .put("ageMs", (nowNs - createdNs) / 1_000_000)
+                .put("configuredMs", ms(createdNs, configuredNs))
+                .put("startedMs", ms(createdNs, startedNs))
+                .put("stoppedMs", ms(createdNs, stoppedNs))
+                .put("releaseCalledMs", ms(createdNs, releaseCalledNs))
+                .put("releasedMs", ms(createdNs, releasedNs))
+                // The gap between asking and finishing. A release that blocks
+                // is a different defect from one that was never called.
+                .put("releaseTookMs", ms(releaseCalledNs, releasedNs))
+                .put("live", live)
+                .also { o -> failure?.let { o.put("failure", it) } }
+
+        private fun ms(from: Long, to: Long): Long =
+            if (from == 0L || to == 0L) -1 else (to - from) / 1_000_000
+    }
+
+    private val entries = java.util.Collections.synchronizedList(ArrayList<Entry>())
+
+    @Volatile var peakLive = 0
+        private set
+
+    /** The job codecs are attributed to, set around a preparation. */
+    @Volatile var currentJob: String = "none"
+
+    fun open(role: String): Entry {
+        val entry = Entry(nextId.incrementAndGet(), role, currentJob, System.nanoTime())
+        entries.add(entry)
+        val live = liveCount()
+        if (live > peakLive) peakLive = live
+        return entry
+    }
+
+    fun liveCount(): Int = entries.count { it.live }
+
+    fun liveEntries(): List<Entry> = entries.filter { it.live }
+
+    /**
+     * Everything recorded, newest last.
+     *
+     * Bounded by trimming completed entries once there are plenty: a
+     * preparation creates seven or eight, and keeping every one from a long
+     * session would grow without limit for no extra evidence.
+     */
+    fun snapshot(): JSONObject {
+        val nowNs = System.nanoTime()
+        val out = JSONArray()
+        synchronized(entries) {
+            for (e in entries) out.put(e.json(nowNs))
+        }
+        return JSONObject()
+            .put("ok", true)
+            .put("live", liveCount())
+            .put("peakLive", peakLive)
+            .put("total", entries.size)
+            .put("entries", out)
+    }
+
+    fun trimCompleted(keep: Int = 32) {
+        synchronized(entries) {
+            val done = entries.filter { !it.live }
+            if (done.size <= keep) return
+            entries.removeAll(done.take(done.size - keep).toSet())
+        }
+    }
+
+    fun reset() {
+        entries.clear()
+        peakLive = 0
+    }
+}