@pictor-sdk/video-android 0.1.0

package/android/src/main/java/dev/pictor/video/PreviewSync.kt

@@ -0,0 +1,374 @@
+package dev.pictor.video
+
+import org.json.JSONObject
+
+/**
+ * Keeping the picture and the mix together through a stall.
+ *
+ * ## What goes wrong without it
+ *
+ * A clip seam makes the video player buffer while it switches decoder. The mix
+ * is one continuous file and has no reason to stop, so it runs ahead by exactly
+ * the length of the stall and keeps the lead — measured at +256 µs·10³ from the
+ * first seam onward, and it never came back.
+ *
+ * Correcting that afterwards does not work. A seek into a compressed mix costs
+ * 200–340 ms on a Pixel 7, which is the same size as the error, so a monitor
+ * aiming at where the picture *was* lands behind it and then chases the other
+ * way. Measured, it oscillated +264, −194, −207, −178 for as long as playback
+ * continued.
+ *
+ * ## What this does instead
+ *
+ * Both transports are held for the stall and released together, aligned.
+ *
+ * - **A stall holds the mix and remembers the intent.** The picture is already
+ *   stopped — that is what a stall is — so only the mix needs holding. The
+ *   user's intent to be playing is kept separately, because a hold is not a
+ *   pause.
+ *
+ *   Holding the picture's own transport was tried first and deadlocked: taking
+ *   `playWhenReady` away stops its position advancing, and an advancing
+ *   position is the only evidence that the stall has ended. One hold, zero
+ *   releases, and playback never resumed.
+ * - **Resume waits for alignment.** When the picture is ready again, the mix's
+ *   position is compared against it. If they already agree, both start. If they
+ *   do not, the mix seeks *while everything is held* — nothing is advancing, so
+ *   the seek has nothing to chase — and the release waits for it to land.
+ * - **Epochs guard it.** A hold belongs to a generation; a ready callback from
+ *   a mix that has since been replaced cannot release a hold that is no longer
+ *   its own.
+ * - **The user always wins.** An explicit pause, a background or an unmount
+ *   clears the intent, and a pending automatic resume is then dropped rather
+ *   than fired later at a surprising moment.
+ */
+internal object PreviewSync {
+
+    /**
+     * How close counts as aligned when releasing a hold.
+     *
+     * Tighter than the drift backstop, because nothing is moving: the mix has
+     * been seeked while held and the only question is whether it landed. A
+     * value near the mix's own access-unit granularity is as good as this can
+     * be without decoding.
+     */
+    private const val ALIGN_US = 30_000L
+
+    /** How long to wait for a seek to land before giving up and releasing. */
+    private const val CONVERGE_TIMEOUT_NS = 1_500_000_000L
+
+    private enum class State { IDLE, HELD, AWAITING_SEEK }
+
+    /**
+     * How long the picture may stop advancing before it counts as stalled.
+     *
+     * Detected by measurement, not by a callback. `onPlaybackStateChanged`
+     * never reported `STATE_BUFFERING` at a clip seam on a Pixel 7 — the
+     * transition is seamless as far as the state machine is concerned — while
+     * the position stopped advancing for about 230 ms and the mix ran on. A
+     * hold keyed on that callback engaged zero times in a 22 s run.
+     *
+     * Two anchor publications is the floor: the pump runs at 50 ms, so one
+     * unchanged reading proves nothing.
+     */
+    private const val STALL_NS = 110_000_000L
+
+    @Volatile private var state = State.IDLE
+
+    /**
+     * Whether the person wants playback to be running.
+     *
+     * Separate from whether it *is* running. A hold stops both transports
+     * without changing this, which is what lets the release know it should
+     * start them again rather than leave them stopped.
+     */
+    @Volatile var intentPlaying = false
+        private set
+
+    /** The mix generation a hold belongs to. */
+    @Volatile private var holdEpoch = 0L
+
+    /** When the realigning seek was issued, for the convergence timeout. */
+    @Volatile private var seekAtNs = 0L
+
+    /** Where the seek aimed, so convergence can be judged against it. */
+    @Volatile private var seekTargetUs = 0L
+
+    /** The last *raw* anchor position, and when it last changed. */
+    @Volatile private var lastRawUs = Long.MIN_VALUE
+    @Volatile private var lastRawChangeNs = 0L
+
+    /** True while the picture is measurably not advancing. */
+    @Volatile var stalled = false
+        private set
+
+    @Volatile var stalls = 0L
+        private set
+
+    /** Counters, so a gate can assert that holds happened and resolved. */
+    @Volatile var holds = 0L
+        private set
+
+    @Volatile var releases = 0L
+        private set
+
+    @Volatile var realigningSeeks = 0L
+        private set
+
+    /** The picture's position, as *presented* rather than as decoded. */
+    fun presentedUs(): Long? {
+        val sample = FrameClock.latestSample() ?: return null
+        val ptsUs = sample[FrameClock.Field.PTS_US]
+        val window = sample[FrameClock.Field.WINDOW_INDEX].toInt()
+        return Media3Player.globalPositionUs(window, ptsUs)
+    }
+
+    /**
+     * The presented frame, with everything needed to say whether it is fresh.
+     *
+     * A presented time that stops changing has three possible causes and they
+     * need different fixes: the renderer drew the same frame again, the record
+     * is stale because nothing new was consumed, or the source timestamp is
+     * being mapped to the wrong global time. Reading the position alone cannot
+     * tell them apart — the frame's own identity can.
+     *
+     * `id` is the canonical sample's sequence, `generation` changes with the
+     * media item, `ptsUs` is the source timestamp and `consumedNs` is when the
+     * renderer latched it, on the same clock as everything else here.
+     */
+    fun presentedRecord(): LongArray? {
+        val sample = FrameClock.latestSample() ?: return null
+        val window = sample[FrameClock.Field.WINDOW_INDEX]
+        return longArrayOf(
+            sample[FrameClock.Field.ID],
+            sample[FrameClock.Field.GENERATION],
+            window,
+            sample[FrameClock.Field.PTS_US],
+            Media3Player.globalPositionUs(window.toInt(), sample[FrameClock.Field.PTS_US]),
+            sample[FrameClock.Field.CONSUMED_NS],
+        )
+    }
+
+    /** The picture's position as the decoder reports it, extrapolated. */
+    fun decodedUs(atNs: Long): Long {
+        val anchor = PlaybackAnchor.latest()
+        return Media3Player.globalPositionUs(anchor.windowIndex, PlaybackAnchor.positionAt(atNs))
+    }
+
+    private fun audioUs(atNs: Long): Long? {
+        val anchor = PreviewAudioPlayer.current()?.anchor ?: return null
+        return if (anchor.playing) anchor.positionUs + (atNs - anchor.monotonicNs) / 1000
+        else anchor.positionUs
+    }
+
+    private fun trace(type: String, extra: JSONObject.() -> Unit = {}) {
+        PreviewTrace.record(type) {
+            val atNs = System.nanoTime()
+            JSONObject()
+                .put("state", state.name)
+                .put("intent", intentPlaying)
+                .put("presentedUs", presentedUs() ?: -1)
+                .put("decodedUs", decodedUs(atNs))
+                .put("audioUs", audioUs(atNs) ?: -1)
+                .put("audioPlaying", PreviewAudioPlayer.current()?.anchor?.playing ?: false)
+                .also(extra)
+        }
+    }
+
+    /** The host is starting playback. */
+    fun onUserPlay() {
+        intentPlaying = true
+        state = State.IDLE
+        trace("sync.userPlay")
+    }
+
+    /**
+     * The host is stopping playback, or the screen is going away.
+     *
+     * Clears the intent, so a hold in flight releases into a stopped state
+     * rather than starting playback the person has already asked to stop.
+     */
+    fun onUserStop(reason: String) {
+        intentPlaying = false
+        state = State.IDLE
+        trace("sync.userStop") { put("reason", reason) }
+    }
+
+    /**
+     * The picture's buffering state changed.
+     *
+     * Kept because it is the earliest signal when it does fire, and it costs
+     * nothing when it does not. It is no longer the only one: see [STALL_NS].
+     */
+    fun onVideoBuffering(buffering: Boolean, holdVideo: (Boolean) -> Unit) {
+        if (!PreviewConfig.coordinator) return
+        if (buffering && intentPlaying) beginHold(holdVideo, "buffering")
+    }
+
+    private fun beginHold(holdVideo: (Boolean) -> Unit, why: String) {
+        if (state != State.IDLE) return
+        val mix = PreviewAudioPlayer.current() ?: return
+        state = State.HELD
+        holdEpoch = mix.epoch
+        holds += 1
+        attempts = 0
+        trace("sync.hold") { put("why", why) }
+        /*
+          The mix only.
+
+          The picture is already stopped — that is what a stall is — and taking
+          its transport away as well removed the very signal the release waits
+          for. `holdVideo` stays in the signature because an explicit stop still
+          needs to reach it.
+        */
+        mix.pause()
+    }
+
+    /** Realignment attempts within one hold, so a chase cannot run forever. */
+    @Volatile private var attempts = 0
+
+    /** At most this many seeks per hold. */
+    private const val MAX_ATTEMPTS = 2
+
+    /**
+     * Called from the player's own anchor pump, so it exists in production
+     * rather than only while a trace is armed.
+     */
+    fun tick(holdVideo: (Boolean) -> Unit) {
+        if (!PreviewConfig.coordinator) return
+        val nowNs = System.nanoTime()
+        val anchor = PlaybackAnchor.latest()
+
+        /*
+          Stall detection, from the raw position.
+
+          `positionAt` extrapolates, so it keeps advancing through a stall and
+          cannot see one. The published value is what actually stops moving.
+        */
+        if (anchor.positionUs != lastRawUs) {
+            lastRawUs = anchor.positionUs
+            lastRawChangeNs = nowNs
+            if (stalled) {
+                stalled = false
+                PreviewTrace.record("sync.stallEnd") {
+                    JSONObject().put("positionUs", anchor.positionUs)
+                }
+            }
+        } else if (
+            intentPlaying && anchor.playing && !stalled &&
+            lastRawChangeNs != 0L && nowNs - lastRawChangeNs > STALL_NS
+        ) {
+            stalled = true
+            stalls += 1
+            PreviewTrace.record("sync.stallStart") {
+                JSONObject()
+                    .put("positionUs", anchor.positionUs)
+                    .put("stalledForMs", (nowNs - lastRawChangeNs) / 1_000_000)
+            }
+            beginHold(holdVideo, "stall")
+        }
+
+        if (state == State.IDLE) return
+        val mix = PreviewAudioPlayer.current() ?: run { state = State.IDLE; return }
+
+        if (!intentPlaying) {
+            // The person stopped while we were holding. Release the picture's
+            // transport but start nothing.
+            state = State.IDLE
+            holdVideo(false)
+            trace("sync.abandon")
+            return
+        }
+        if (mix.epoch != holdEpoch) {
+            // A new mix arrived mid-hold; the hold was about the old one.
+            state = State.IDLE
+            holdVideo(false)
+            trace("sync.staleHold")
+            return
+        }
+
+        // Still stalled, or the decoder says it is refilling: nothing to align
+        // against yet.
+        if (stalled || anchor.buffering) return
+
+        val atNs = nowNs
+        val videoUs = decodedUs(atNs)
+        val audio = audioUs(atNs) ?: return
+
+        when (state) {
+            State.HELD -> {
+                val gap = audio - videoUs
+                if (Math.abs(gap) <= ALIGN_US || attempts >= MAX_ATTEMPTS) {
+                    release(mix, holdVideo, gap, if (attempts >= MAX_ATTEMPTS) "attemptsSpent" else "aligned")
+                } else {
+                    /*
+                      Seek while the mix is held.
+
+                      It is paused, so the seek has nothing to chase on its own
+                      side — the only thing still moving is the picture, and the
+                      convergence check below re-reads it rather than assuming
+                      the target is still current.
+                    */
+                    state = State.AWAITING_SEEK
+                    seekAtNs = atNs
+                    seekTargetUs = videoUs
+                    attempts += 1
+                    realigningSeeks += 1
+                    trace("sync.realign") { put("gapUs", gap).put("toUs", videoUs).put("attempt", attempts) }
+                    mix.seekTo(videoUs, mix.epoch)
+                }
+            }
+            State.AWAITING_SEEK -> {
+                val landed = Math.abs(audio - seekTargetUs) <= ALIGN_US
+                val timedOut = atNs - seekAtNs > CONVERGE_TIMEOUT_NS
+                if (timedOut) {
+                    release(mix, holdVideo, audio - videoUs, "timeout")
+                } else if (landed) {
+                    // Back to HELD rather than straight to release: the picture
+                    // moved while the seek ran, so the gap is re-measured and
+                    // may warrant one more attempt — bounded by MAX_ATTEMPTS.
+                    state = State.HELD
+                }
+            }
+            State.IDLE -> Unit
+        }
+    }
+
+    private fun release(
+        mix: PreviewAudioPlayer,
+        holdVideo: (Boolean) -> Unit,
+        gapUs: Long,
+        why: String,
+    ) {
+        state = State.IDLE
+        releases += 1
+        attempts = 0
+        trace("sync.release") { put("gapUs", gapUs).put("why", why) }
+        // The picture was never held, so there is nothing to give back to it;
+        // the call is kept so an explicit stop can still reach the transport.
+        holdVideo(false)
+        mix.play(mix.epoch)
+    }
+
+    fun counters(): JSONObject =
+        JSONObject()
+            .put("holds", holds)
+            .put("releases", releases)
+            .put("realigningSeeks", realigningSeeks)
+            .put("stalls", stalls)
+            .put("state", state.name)
+            .put("intentPlaying", intentPlaying)
+
+    fun reset() {
+        state = State.IDLE
+        intentPlaying = false
+        holds = 0
+        releases = 0
+        realigningSeeks = 0
+        attempts = 0
+        stalls = 0
+        stalled = false
+        lastRawUs = Long.MIN_VALUE
+        lastRawChangeNs = 0
+    }
+}