@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/PictorVideoPackage.kt

@@ -0,0 +1,26 @@
+package dev.pictor.video
+
+import com.facebook.react.BaseReactPackage
+import com.facebook.react.bridge.NativeModule
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.module.model.ReactModuleInfo
+import com.facebook.react.module.model.ReactModuleInfoProvider
+
+/** Autolinked entry point. Android only; iOS is deferred by design. */
+class PictorVideoPackage : BaseReactPackage() {
+  override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? =
+    if (name == PictorVideoModule.NAME) PictorVideoModule(reactContext) else null
+
+  override fun getReactModuleInfoProvider() = ReactModuleInfoProvider {
+    mapOf(
+      PictorVideoModule.NAME to ReactModuleInfo(
+        PictorVideoModule.NAME,
+        PictorVideoModule.NAME,
+        false, // canOverrideExistingModule
+        false, // needsEagerInit
+        false, // isCxxModule
+        true,  // isTurboModule
+      ),
+    )
+  }
+}

package/android/src/main/java/dev/pictor/video/PlaybackAnchor.kt

@@ -0,0 +1,161 @@
+package dev.pictor.video
+
+/**
+ * The player's position, sampled on its own thread and extrapolated elsewhere.
+ *
+ * `ExoPlayer` may only be read from the application thread, and measuring
+ * synchronisation with a `post` and a latch puts a thread hop *inside the
+ * measurement*: the position arrives from later than the frame it is compared
+ * against. So the position is published as an anchor — `(positionUs,
+ * monotonicNs, rate)` — and readers extrapolate.
+ *
+ * **The anchor is only as good as its granularity.** `Player.currentPosition`
+ * does not advance continuously; it is itself derived, and between its updates
+ * it can return a value already behind real time. Extrapolating forward from a
+ * stale value yields a position that is too low, which shows up as a constant
+ * positive bias in any offset computed against it. [granularity] measures that
+ * directly rather than leaving it as a suspicion — how often the value changes,
+ * how big its steps are, and how old the anchor was when it was used.
+ *
+ * Lock-free: one immutable [Sample] behind a volatile reference. Readers on the
+ * render path never block and never see a half-written value.
+ */
+internal object PlaybackAnchor {
+
+    /** `System.nanoTime`, or a test's clock. Read through [nowNs]. */
+    @Volatile
+    var clock: () -> Long = { System.nanoTime() }
+
+    fun nowNs(): Long = clock()
+
+    /**
+     * An instant of player state. Immutable; replaced, never mutated.
+     *
+     * @param positionUs media position at [monotonicNs]
+     * @param monotonicNs when the position was read, on the same clock as
+     *   `VideoFrameMetadataListener`'s `releaseTimeNs`
+     * @param rate playback speed, so extrapolation tracks 0.5× and 2×
+     * @param playing whether the clock is advancing at all
+     * @param buffering a stall, during which extrapolation is meaningless
+     * @param discontinuity a seek or source change has invalidated what came
+     *   before
+     */
+    data class Sample(
+        val positionUs: Long,
+        val monotonicNs: Long,
+        val rate: Float,
+        val playing: Boolean,
+        val buffering: Boolean,
+        val discontinuity: Boolean,
+        val generation: Int,
+        /**
+         * The playlist window [positionUs] is measured within.
+         *
+         * Carried because the position is window-local: a reader converting to
+         * project time needs to know which window it belongs to, and reading
+         * that separately would pair a position with an index from a different
+         * instant.
+         */
+        val windowIndex: Int = 0,
+    )
+
+    /** What the anchor's own resolution is, measured rather than assumed. */
+    data class Granularity(
+        /** Anchor publications so far. */
+        val publications: Long,
+        /** Publications where the position actually moved. */
+        val changes: Long,
+        /** Largest observed jump between consecutive distinct positions, µs. */
+        val maxStepUs: Long,
+        /** Sum of steps, for a mean. */
+        val totalStepUs: Long,
+        /** Longest the position stayed unchanged, in nanoseconds. */
+        val maxStaleNs: Long,
+    )
+
+    private val EMPTY = Sample(0, 0, 1f, false, false, false, 0, 0)
+
+    @Volatile
+    private var current = EMPTY
+
+    @Volatile
+    private var publications = 0L
+
+    @Volatile
+    private var changes = 0L
+
+    @Volatile
+    private var maxStepUs = 0L
+
+    @Volatile
+    private var totalStepUs = 0L
+
+    @Volatile
+    private var maxStaleNs = 0L
+
+    @Volatile
+    private var lastChangeNs = 0L
+
+    @JvmStatic
+    fun publish(sample: Sample) {
+        val previous = current
+        publications += 1
+        if (sample.positionUs != previous.positionUs) {
+            changes += 1
+            val step = kotlin.math.abs(sample.positionUs - previous.positionUs)
+            if (previous.monotonicNs != 0L) {
+                totalStepUs += step
+                if (step > maxStepUs) maxStepUs = step
+                val stale = sample.monotonicNs - lastChangeNs
+                if (lastChangeNs != 0L && stale > maxStaleNs) maxStaleNs = stale
+            }
+            lastChangeNs = sample.monotonicNs
+        }
+        current = sample
+    }
+
+    @JvmStatic
+    fun latest(): Sample = current
+
+    @JvmStatic
+    fun granularity(): Granularity =
+        Granularity(publications, changes, maxStepUs, totalStepUs, maxStaleNs)
+
+    /** How old the anchor was at [atNs], in nanoseconds. */
+    @JvmStatic
+    fun ageNs(atNs: Long): Long {
+        val anchor = current
+        if (anchor.monotonicNs == 0L) return 0
+        return atNs - anchor.monotonicNs
+    }
+
+    /**
+     * Where the playhead is at [atMonotonicNs], from the newest anchor.
+     *
+     * Returns the anchor unextrapolated when paused or buffering: the clock is
+     * not advancing then, and pretending otherwise would invent a position.
+     * Extrapolating backwards is allowed — a frame's release time can precede
+     * the newest anchor — because refusing would bias the result toward late.
+     */
+    @JvmStatic
+    fun positionAt(atMonotonicNs: Long): Long {
+        val anchor = current
+        if (!anchor.playing || anchor.buffering || anchor.monotonicNs == 0L) {
+            return anchor.positionUs
+        }
+        val elapsedNs = atMonotonicNs - anchor.monotonicNs
+        return anchor.positionUs + (elapsedNs / 1000.0 * anchor.rate).toLong()
+    }
+
+    @JvmStatic
+    fun reset() {
+        current = EMPTY
+        publications = 0
+        changes = 0
+        maxStepUs = 0
+        totalStepUs = 0
+        maxStaleNs = 0
+        lastChangeNs = 0
+        clock = { System.nanoTime() }
+    }
+}

package/android/src/main/java/dev/pictor/video/PreviewAudioPlayer.kt

@@ -0,0 +1,291 @@
+package dev.pictor.video
+
+import android.content.Context
+import android.net.Uri
+import android.os.Handler
+import android.os.Looper
+import androidx.media3.common.MediaItem
+import androidx.media3.common.PlaybackException
+import androidx.media3.common.Player
+import androidx.media3.exoplayer.ExoPlayer
+import org.json.JSONObject
+
+/**
+ * Plays one prepared mix, and nothing else.
+ *
+ * ## Why a second player rather than a second track on the first
+ *
+ * The video player renders into a persistent external-OES surface and carries a
+ * playlist of clips, each with its own speed. The mix is a single continuous
+ * file covering the whole timeline, already re-timed — its position *is* the
+ * global timeline position, with no window index and no per-item rate. Feeding
+ * it to the video player would mean a playlist whose items disagree about what
+ * their positions mean.
+ *
+ * So this is deliberately small: one item, no surface, no playlist.
+ *
+ * ## Timeline position, not player position
+ *
+ * A caller asks for a global timeline microsecond and gets one back. That is
+ * the only coordinate the two players share, and it is what makes drift
+ * measurable — [state] reports the mix's own position alongside the wall clock
+ * it was sampled at, so a monitor compares two timestamps rather than
+ * accumulating deltas that drift by construction.
+ *
+ * ## Nothing here interprets the document
+ *
+ * No gain, no fades, no speed. All of that was compiled into the file by
+ * [AudioMixer] from the same program an export uses. This player's whole job is
+ * to be at the right position.
+ */
+internal class PreviewAudioPlayer private constructor(private val player: ExoPlayer) {
+
+    @Volatile private var released = false
+
+    /** Everything ExoPlayer is touched from is its own application thread. */
+    private val main = Handler(Looper.getMainLooper())
+
+    /**
+     * The mix's position, published rather than fetched.
+     *
+     * `ExoPlayer` may only be read from the application thread, so a caller
+     * asking from anywhere else pays a thread hop *inside the measurement* —
+     * the position arrives from later than the moment it is compared against.
+     * The video player already solves this by publishing an anchor; the mix
+     * publishes one on the same clock so the two can simply be subtracted.
+     */
+    @Volatile var anchor: Anchor = Anchor(0, 0, false, false, 0, 1f)
+        private set
+
+    data class Anchor(
+        val positionUs: Long,
+        val monotonicNs: Long,
+        val playing: Boolean,
+        val ready: Boolean,
+        val epoch: Long,
+        /**
+         * The speed this player is actually running at.
+         *
+         * Published rather than assumed. The mix covers global project time
+         * with every clip's speed already compiled in, so it must run at the
+         * preview rate alone — never at the current clip's 2x or 0.5x. An
+         * anchor extrapolated at 1.0 while the player runs at anything else
+         * reports an offset that is partly its own arithmetic, which is the
+         * one kind of error a synchronisation gate cannot see.
+         */
+        val rate: Float,
+    )
+
+    /** One re-posted Runnable, never a fresh lambda. */
+    private val anchorPump = object : Runnable {
+        override fun run() {
+            if (released) return
+            anchor = Anchor(
+                positionUs = player.currentPosition * 1000,
+                monotonicNs = System.nanoTime(),
+                playing = player.isPlaying,
+                ready = player.playbackState == Player.STATE_READY,
+                epoch = epoch,
+                rate = player.playbackParameters.speed,
+            )
+            // The same cadence the video player's anchor uses. 25 ms was
+            // chosen for resolution and is being tested as a suspect for the
+            // renderer stall: two players posting to one looper is twice the
+            // main-thread traffic the editor had before music existed.
+            // Off in the bisect configuration that excludes it. The pump is
+            // one of four things music adds to the main looper.
+            if (PreviewConfig.positionPump) main.postDelayed(this, 50)
+        }
+    }
+
+    @Volatile private var lastError: String? = null
+
+    /**
+     * The epoch this player belongs to.
+     *
+     * A caller stamps every start and seek. A callback or a state read carrying
+     * an older epoch is from a mix that has since been replaced, and acting on
+     * it would move the picture to match audio that is no longer playing.
+     */
+    @Volatile var epoch: Long = 0
+        private set
+
+    private fun onPlayer(block: ExoPlayer.() -> Unit) {
+        if (released) return
+        if (Looper.myLooper() == Looper.getMainLooper()) player.block()
+        else main.post { if (!released) player.block() }
+    }
+
+    fun load(uriString: String, epoch: Long) {
+        this.epoch = epoch
+        lastError = null
+        PreviewTrace.record("audio.load") { JSONObject().put("epoch", epoch) }
+        // A new file has its own seek cost; carrying the old estimate forward
+        // would aim the first correction at the wrong place.
+        PreviewTrace.resetDrift()
+        onPlayer {
+            setMediaItem(MediaItem.fromUri(Uri.parse(uriString)))
+            prepare()
+        }
+    }
+
+    fun play(epoch: Long) {
+        if (!PreviewConfig.startMix) return
+        // A start for an epoch that has already been superseded is dropped
+        // rather than played: the alternative is audio from the previous edit.
+        PreviewTrace.record("audio.play") {
+            JSONObject().put("epoch", epoch).put("current", this.epoch)
+                .put("accepted", epoch == this.epoch)
+        }
+        if (epoch != this.epoch) return
+        onPlayer { play() }
+    }
+
+    fun pause() {
+        PreviewTrace.mark("audio.pause")
+        onPlayer { pause() }
+    }
+
+    fun seekTo(positionUs: Long, epoch: Long) {
+        PreviewTrace.record("audio.seek") {
+            JSONObject().put("toUs", positionUs).put("epoch", epoch)
+                .put("current", this.epoch).put("accepted", epoch == this.epoch)
+        }
+        if (epoch != this.epoch) return
+        onPlayer { seekTo(positionUs / 1000) }
+    }
+
+    /**
+     * Playback rate.
+     *
+     * Normally 1: clip speed is already compiled into the mix, so re-applying
+     * it here would apply it twice. It exists for the preview-rate control,
+     * which is a property of watching rather than of the document.
+     */
+    fun setRate(rate: Float) {
+        onPlayer { setPlaybackSpeed(if (rate > 0f) rate else 1f) }
+    }
+
+    /**
+     * Where the mix is, and when that was true.
+     *
+     * `monotonicNs` is sampled beside the position on purpose. A drift monitor
+     * that compares two positions read at unknown times is measuring its own
+     * scheduling jitter as much as anything else.
+     */
+    fun state(): String {
+        val out = JSONObject().put("epoch", epoch).put("released", released)
+        if (released) return out.put("ok", false).toString()
+        val read = {
+            out.put("ok", true)
+                .put("positionUs", player.currentPosition * 1000)
+                .put("monotonicNs", System.nanoTime())
+                .put("playing", player.isPlaying)
+                .put("ready", player.playbackState == Player.STATE_READY)
+                .put("ended", player.playbackState == Player.STATE_ENDED)
+                .put("bufferedUs", player.bufferedPosition * 1000)
+            lastError?.let { out.put("error", it) }
+            Unit
+        }
+        if (Looper.myLooper() == Looper.getMainLooper()) {
+            read()
+        } else {
+            // A state read from the JS thread has to hop and wait: reading
+            // ExoPlayer off its own thread is undefined, and a stale cached
+            // value is exactly what a drift monitor must not be given.
+            val latch = java.util.concurrent.CountDownLatch(1)
+            main.post {
+                if (!released) runCatching { read() }
+                latch.countDown()
+            }
+            if (!latch.await(50, java.util.concurrent.TimeUnit.MILLISECONDS)) {
+                out.put("ok", false).put("error", "timed out reading player state")
+            }
+        }
+        return out.toString()
+    }
+
+    fun release() {
+        if (released) return
+        PreviewTrace.mark("audio.release")
+        released = true
+        // On the player's own thread, explicitly. Nothing native is reachable
+        // from a finalizer here, for the reason the video binding documents.
+        if (Looper.myLooper() == Looper.getMainLooper()) player.release()
+        else main.post { player.release() }
+    }
+
+    companion object {
+        @Volatile private var instance: PreviewAudioPlayer? = null
+
+        /** The one preview-audio player, created on demand. */
+        /**
+         * Players ever built, players released, and how many are alive now.
+         *
+         * A rebuild replaces the mix, and the question a bounds gate asks is
+         * whether it also replaces the *player*. One alive at all times is the
+         * design; the counters are what turn that from an argument about a
+         * singleton into a measurement.
+         */
+        @Volatile var created = 0L
+            private set
+
+        @Volatile var releasedCount = 0L
+            private set
+
+        fun liveCount(): Int = if (instance?.released == false) 1 else 0
+
+        fun instanceCounters(): org.json.JSONObject =
+            org.json.JSONObject()
+                .put("created", created)
+                .put("released", releasedCount)
+                .put("live", liveCount())
+
+        fun require(context: Context): PreviewAudioPlayer {
+            instance?.let { if (!it.released) return it }
+            val built = build(context)
+            created += 1
+            instance = built
+            return built
+        }
+
+        fun current(): PreviewAudioPlayer? = instance?.takeIf { !it.released }
+
+        fun releaseAll() {
+            if (instance?.released == false) releasedCount += 1
+            instance?.release()
+            instance = null
+        }
+
+        private fun build(context: Context): PreviewAudioPlayer {
+            val latch = java.util.concurrent.CountDownLatch(1)
+            var built: PreviewAudioPlayer? = null
+            val make = {
+                val exo = ExoPlayer.Builder(context).build()
+                val wrapper = PreviewAudioPlayer(exo)
+                exo.addListener(
+                    object : Player.Listener {
+                        override fun onPlayerError(error: PlaybackException) {
+                            // Recorded rather than thrown: a mix that will not
+                            // open is a state the coordinator has to see, not an
+                            // exception on somebody else's thread.
+                            wrapper.lastError = error.errorCodeName + ": " + error.message
+                        }
+                    },
+                )
+                built = wrapper
+                // Publishing starts with the player and stops with it: an
+                // anchor from a released player is a position that no longer
+                // exists.
+                wrapper.main.post(wrapper.anchorPump)
+                latch.countDown()
+            }
+            if (Looper.myLooper() == Looper.getMainLooper()) make()
+            else {
+                Handler(Looper.getMainLooper()).post(make)
+                latch.await(2, java.util.concurrent.TimeUnit.SECONDS)
+            }
+            return built ?: throw IllegalStateException("preview audio player could not be created")
+        }
+    }
+}

package/android/src/main/java/dev/pictor/video/PreviewConfig.kt

@@ -0,0 +1,96 @@
+package dev.pictor.video
+
+import org.json.JSONObject
+
+/**
+ * Which parts of the music preview exist, for bisecting a failure.
+ *
+ * The picture stops advancing when music is active and the counters say the
+ * *consumer* stopped: `available` reaches three and stays there while the
+ * decoder keeps offering. Three availability callbacks and a stalled consumer
+ * is what a full BufferQueue looks like — nothing latches, so nothing is
+ * recycled, so the producer has nowhere to put frame four.
+ *
+ * That narrows it to the render path, and the music preview added four things
+ * that could reach it: a second `ExoPlayer`, its position pump on the main
+ * looper, the stall coordinator that ticks from the video player's own pump,
+ * and the mix actually playing. Turning them off one at a time is the only way
+ * to say which, and guessing between them is how the last three "fixes" in this
+ * area were tuning rather than repair.
+ *
+ * Every flag defaults to on, so a build without diagnostics behaves exactly as
+ * production does. They are settable only through the diagnostics-gated module
+ * method, and the harness persists a choice so a *fresh process* can apply it —
+ * an intermittent failure re-measured inside a warm process is not the same
+ * experiment.
+ */
+internal object PreviewConfig {
+
+    /** Create and load the second player at all. */
+    @Volatile var loadPlayer = true
+
+    /** Let the mix actually start when playback starts. */
+    @Volatile var startMix = true
+
+    /** Publish the mix's position anchor from the main looper. */
+    @Volatile var positionPump = true
+
+    /** Run the stall coordinator. */
+    @Volatile var coordinator = true
+
+    /**
+     * Let the drift check act on what it finds.
+     *
+     * The controlled comparison for correction-seek interference. The trace
+     * implicates it directly: at a seam the project clock runs at half rate for
+     * ~450 ms while the player applies a speed change it has already reported,
+     * a correction fires on that wrong reading, and the overshoot it leaves is
+     * larger and longer-lived than the excursion it was answering. Turning the
+     * correction off — while still measuring and recording what it *would* have
+     * done — separates the excursion from the response to it.
+     */
+    @Volatile var driftCorrection = true
+
+    /**
+     * Keep a copy of the mixed PCM for the attribution gate.
+     *
+     * Off by default. A 33-second stereo mix is 12.7 MB and this held one for
+     * the life of the process, in addition to the array being encoded — a cost
+     * paid by every run after the one experiment that wanted it.
+     */
+    @Volatile var retainMixedPcm = false
+
+    fun apply(json: String): JSONObject {
+        if (!Diagnostics.ENABLED) return JSONObject(Diagnostics.unavailable())
+        return try {
+            val o = JSONObject(json)
+            if (o.has("loadPlayer")) loadPlayer = o.getBoolean("loadPlayer")
+            if (o.has("startMix")) startMix = o.getBoolean("startMix")
+            if (o.has("positionPump")) positionPump = o.getBoolean("positionPump")
+            if (o.has("coordinator")) coordinator = o.getBoolean("coordinator")
+            if (o.has("driftCorrection")) driftCorrection = o.getBoolean("driftCorrection")
+            if (o.has("retainMixedPcm")) retainMixedPcm = o.getBoolean("retainMixedPcm")
+            current().put("ok", true)
+        } catch (e: Exception) {
+            JSONObject().put("ok", false).put("error", e.javaClass.simpleName)
+        }
+    }
+
+    fun current(): JSONObject =
+        JSONObject()
+            .put("loadPlayer", loadPlayer)
+            .put("startMix", startMix)
+            .put("positionPump", positionPump)
+            .put("coordinator", coordinator)
+            .put("driftCorrection", driftCorrection)
+            .put("retainMixedPcm", retainMixedPcm)
+
+    fun reset() {
+        loadPlayer = true
+        startMix = true
+        positionPump = true
+        coordinator = true
+        driftCorrection = true
+        retainMixedPcm = false
+    }
+}